GH-103082: Filter LINE events in VM, to simplify tool implementation. (GH-104387)

When monitoring LINE events, instrument all instructions that can have a predecessor on a different line.
Then check that the a new line has been hit in the instrumentation code.
This brings the behavior closer to that of 3.11, simplifying implementation and porting of tools.

Include/internal/pycore_frame.h

@@ -19,7 +19,6 @@ struct _frame {
     struct _PyInterpreterFrame *f_frame; /* points to the frame data */
     PyObject *f_trace;          /* Trace function */
     int f_lineno;               /* Current line number. Only valid if non-zero */
-    int f_last_traced_line;     /* The last line traced for this frame */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
     char f_fast_as_locals;      /* Have the fast locals of this frame been converted to a dict? */

Include/internal/pycore_instruments.h

@@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event,
 
 extern int
 _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
-                              _Py_CODEUNIT *instr);
+                              _Py_CODEUNIT *instr, _Py_CODEUNIT *prev);
 
 extern int
 _Py_call_instrumentation_instruction(
     PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr);
 
-int
+_Py_CODEUNIT *
 _Py_call_instrumentation_jump(
     PyThreadState *tstate, int event,
     _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target);
@@ -100,6 +100,7 @@ extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, int index);
 
 extern PyObject _PyInstrumentation_MISSING;
+extern PyObject _PyInstrumentation_DISABLE;
 
 #ifdef __cplusplus
 }

Lib/test/test_monitoring.py

@@ -524,7 +524,7 @@ class LineMonitoringTest(MonitoringTestBase, unittest.TestCase):
             sys.monitoring.set_events(TEST_TOOL, 0)
             sys.monitoring.register_callback(TEST_TOOL, E.LINE, None)
             start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno
-            self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8])
+            self.assertEqual(events, [start+7, 21, 22, 21, 22, 21, start+8])
         finally:
             sys.monitoring.set_events(TEST_TOOL, 0)
             sys.monitoring.register_callback(TEST_TOOL, E.LINE, None)
@@ -1050,6 +1050,8 @@ class TestLocalEvents(MonitoringTestBase, unittest.TestCase):
 def line_from_offset(code, offset):
     for start, end, line in code.co_lines():
         if start <= offset < end:
+            if line is None:
+                return f"[offset={offset}]"
             return line - code.co_firstlineno
     return -1
 
@@ -1072,9 +1074,20 @@ class BranchRecorder(JumpRecorder):
     event_type = E.BRANCH
     name = "branch"
 
+class ReturnRecorder:
+
+    event_type = E.PY_RETURN
+
+    def __init__(self, events):
+        self.events = events
+
+    def __call__(self, code, offset, val):
+        self.events.append(("return", val))
+
 
 JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder
 JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder
+FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder
 
 class TestBranchAndJumpEvents(CheckEvents):
     maxDiff = None
@@ -1098,7 +1111,6 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('jump', 'func', 4, 2),
             ('branch', 'func', 2, 2)])
 
-
         self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
             ('line', 'check_events', 10),
             ('line', 'func', 1),
@@ -1108,15 +1120,62 @@ class TestBranchAndJumpEvents(CheckEvents):
             ('branch', 'func', 3, 6),
             ('line', 'func', 6),
             ('jump', 'func', 6, 2),
+            ('line', 'func', 2),
             ('branch', 'func', 2, 2),
             ('line', 'func', 3),
             ('branch', 'func', 3, 4),
             ('line', 'func', 4),
             ('jump', 'func', 4, 2),
-            ('branch', 'func', 2, 2),
             ('line', 'func', 2),
+            ('branch', 'func', 2, 2),
             ('line', 'check_events', 11)])
 
+    def test_except_star(self):
+
+        class Foo:
+            def meth(self):
+                pass
+
+        def func():
+            try:
+                try:
+                    raise KeyError
+                except* Exception as e:
+                    f = Foo(); f.meth()
+            except KeyError:
+                pass
+
+
+        self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
+            ('line', 'check_events', 10),
+            ('line', 'func', 1),
+            ('line', 'func', 2),
+            ('line', 'func', 3),
+            ('line', 'func', 4),
+            ('branch', 'func', 4, 4),
+            ('line', 'func', 5),
+            ('line', 'meth', 1),
+            ('jump', 'func', 5, 5),
+            ('jump', 'func', 5, '[offset=114]'),
+            ('branch', 'func', '[offset=120]', '[offset=122]'),
+            ('line', 'check_events', 11)])
+
+        self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [
+            ('line', 'check_events', 10),
+            ('line', 'func', 1),
+            ('line', 'func', 2),
+            ('line', 'func', 3),
+            ('raise', KeyError),
+            ('line', 'func', 4),
+            ('branch', 'func', 4, 4),
+            ('line', 'func', 5),
+            ('line', 'meth', 1),
+            ('return', None),
+            ('jump', 'func', 5, 5),
+            ('jump', 'func', 5, '[offset=114]'),
+            ('branch', 'func', '[offset=120]', '[offset=122]'),
+            ('return', None),
+            ('line', 'check_events', 11)])
 
 class TestSetGetEvents(MonitoringTestBase, unittest.TestCase):
 

Lib/test/test_pdb.py

@@ -1793,8 +1793,9 @@ def test_pdb_issue_gh_101517():
     ...     'continue'
     ... ]):
     ...    test_function()
-    > <doctest test.test_pdb.test_pdb_issue_gh_101517[0]>(5)test_function()
+    --Return--
-    -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace()
+    > <doctest test.test_pdb.test_pdb_issue_gh_101517[0]>(None)test_function()->None
+    -> Warning: lineno is None
     (Pdb) continue
     """
 

Lib/test/test_sys.py

@@ -1446,7 +1446,7 @@ class SizeofTest(unittest.TestCase):
         def func():
             return sys._getframe()
         x = func()
-        check(x, size('3Pii3c7P2ic??2P'))
+        check(x, size('3Pi3c7P2ic??2P'))
         # function
         def func(): pass
         check(func, size('14Pi'))

Lib/test/test_sys_settrace.py

@@ -2867,6 +2867,5 @@ class TestSetLocalTrace(TraceTestCase):
         sys.settrace(None)
 
 
-
 if __name__ == "__main__":
     unittest.main()

Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst

@@ -0,0 +1,5 @@
+Change behavior of ``sys.monitoring.events.LINE`` events in
+``sys.monitoring``: Line events now occur when a new line is reached
+dynamically, instead of using a static approximation, as before. This makes
+the behavior very similar to that of "line" events in ``sys.settrace``. This
+should ease porting of tools from 3.11 to 3.12.

Objects/frameobject.c

@@ -831,7 +831,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
         start_stack = pop_value(start_stack);
     }
     /* Finally set the new lasti and return OK. */
-    f->f_last_traced_line = new_lineno;
     f->f_lineno = 0;
     f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr;
     return 0;
@@ -854,7 +853,6 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure)
     }
     if (v != f->f_trace) {
         Py_XSETREF(f->f_trace, Py_XNewRef(v));
-        f->f_last_traced_line = -1;
     }
     return 0;
 }
@@ -1056,7 +1054,6 @@ _PyFrame_New_NoTrack(PyCodeObject *code)
     f->f_trace_opcodes = 0;
     f->f_fast_as_locals = 0;
     f->f_lineno = 0;
-    f->f_last_traced_line = -1;
     return f;
 }
 

Python/bytecodes.c

@@ -3288,28 +3288,6 @@ dummy_func(
             assert(oparg >= 2);
         }
 
-        inst(INSTRUMENTED_LINE, ( -- )) {
-            _Py_CODEUNIT *here = next_instr-1;
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            int original_opcode = _Py_call_instrumentation_line(
-                    tstate, frame, here);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            if (original_opcode < 0) {
-                next_instr = here+1;
-                goto error;
-            }
-            next_instr = frame->prev_instr;
-            if (next_instr != here) {
-                DISPATCH();
-            }
-            if (_PyOpcode_Caches[original_opcode]) {
-                _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
-                INCREMENT_ADAPTIVE_COUNTER(cache->counter);
-            }
-            opcode = original_opcode;
-            DISPATCH_GOTO();
-        }
-
         inst(INSTRUMENTED_INSTRUCTION, ( -- )) {
             int next_opcode = _Py_call_instrumentation_instruction(
                 tstate, frame, next_instr-1);

Python/ceval.c

@@ -775,6 +775,41 @@ handle_eval_breaker:
 
 #include "generated_cases.c.h"
 
+    /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c,
+     * because it needs to capture frame->prev_instr before it is updated,
+     * as happens in the standard instruction prologue.
+     */
+#if USE_COMPUTED_GOTOS
+        TARGET_INSTRUMENTED_LINE:
+#else
+        case INSTRUMENTED_LINE:
+#endif
+    {
+        _Py_CODEUNIT *prev = frame->prev_instr;
+        _Py_CODEUNIT *here = frame->prev_instr = next_instr;
+        _PyFrame_SetStackPointer(frame, stack_pointer);
+        int original_opcode = _Py_call_instrumentation_line(
+                tstate, frame, here, prev);
+        stack_pointer = _PyFrame_GetStackPointer(frame);
+        if (original_opcode < 0) {
+            next_instr = here+1;
+            goto error;
+        }
+        next_instr = frame->prev_instr;
+        if (next_instr != here) {
+            DISPATCH();
+        }
+        if (_PyOpcode_Caches[original_opcode]) {
+            _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
+            /* Prevent the underlying instruction from specializing
+             * and overwriting the instrumentation. */
+            INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+        }
+        opcode = original_opcode;
+        DISPATCH_GOTO();
+    }
+
+
 #if USE_COMPUTED_GOTOS
         _unknown_opcode:
 #else

Python/ceval_macros.h

@@ -334,11 +334,10 @@ do { \
 #define INSTRUMENTED_JUMP(src, dest, event) \
 do { \
     _PyFrame_SetStackPointer(frame, stack_pointer); \
-    int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \
+    next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \
     stack_pointer = _PyFrame_GetStackPointer(frame); \
-    if (err) { \
+    if (next_instr == NULL) { \
         next_instr = (dest)+1; \
         goto error; \
     } \
-    next_instr = frame->prev_instr; \
 } while (0);

Python/generated_cases.c.h

@@ -4568,32 +4568,8 @@
             DISPATCH();
         }
 
-        TARGET(INSTRUMENTED_LINE) {
-            #line 3292 "Python/bytecodes.c"
-            _Py_CODEUNIT *here = next_instr-1;
-            _PyFrame_SetStackPointer(frame, stack_pointer);
-            int original_opcode = _Py_call_instrumentation_line(
-                    tstate, frame, here);
-            stack_pointer = _PyFrame_GetStackPointer(frame);
-            if (original_opcode < 0) {
-                next_instr = here+1;
-                goto error;
-            }
-            next_instr = frame->prev_instr;
-            if (next_instr != here) {
-                DISPATCH();
-            }
-            if (_PyOpcode_Caches[original_opcode]) {
-                _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
-                INCREMENT_ADAPTIVE_COUNTER(cache->counter);
-            }
-            opcode = original_opcode;
-            DISPATCH_GOTO();
-            #line 4593 "Python/generated_cases.c.h"
-        }
-
         TARGET(INSTRUMENTED_INSTRUCTION) {
-            #line 3314 "Python/bytecodes.c"
+            #line 3292 "Python/bytecodes.c"
             int next_opcode = _Py_call_instrumentation_instruction(
                 tstate, frame, next_instr-1);
             if (next_opcode < 0) goto error;
@@ -4605,26 +4581,26 @@
             assert(next_opcode > 0 && next_opcode < 256);
             opcode = next_opcode;
             DISPATCH_GOTO();
-            #line 4609 "Python/generated_cases.c.h"
+            #line 4585 "Python/generated_cases.c.h"
         }
 
         TARGET(INSTRUMENTED_JUMP_FORWARD) {
-            #line 3328 "Python/bytecodes.c"
+            #line 3306 "Python/bytecodes.c"
             INSTRUMENTED_JUMP(next_instr-1, next_instr+oparg, PY_MONITORING_EVENT_JUMP);
-            #line 4615 "Python/generated_cases.c.h"
+            #line 4591 "Python/generated_cases.c.h"
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_JUMP_BACKWARD) {
-            #line 3332 "Python/bytecodes.c"
+            #line 3310 "Python/bytecodes.c"
             INSTRUMENTED_JUMP(next_instr-1, next_instr-oparg, PY_MONITORING_EVENT_JUMP);
-            #line 4622 "Python/generated_cases.c.h"
+            #line 4598 "Python/generated_cases.c.h"
             CHECK_EVAL_BREAKER();
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_TRUE) {
-            #line 3337 "Python/bytecodes.c"
+            #line 3315 "Python/bytecodes.c"
             PyObject *cond = POP();
             int err = PyObject_IsTrue(cond);
             Py_DECREF(cond);
@@ -4633,12 +4609,12 @@
             assert(err == 0 || err == 1);
             int offset = err*oparg;
             INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            #line 4637 "Python/generated_cases.c.h"
+            #line 4613 "Python/generated_cases.c.h"
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_FALSE) {
-            #line 3348 "Python/bytecodes.c"
+            #line 3326 "Python/bytecodes.c"
             PyObject *cond = POP();
             int err = PyObject_IsTrue(cond);
             Py_DECREF(cond);
@@ -4647,12 +4623,12 @@
             assert(err == 0 || err == 1);
             int offset = (1-err)*oparg;
             INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            #line 4651 "Python/generated_cases.c.h"
+            #line 4627 "Python/generated_cases.c.h"
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_NONE) {
-            #line 3359 "Python/bytecodes.c"
+            #line 3337 "Python/bytecodes.c"
             PyObject *value = POP();
             _Py_CODEUNIT *here = next_instr-1;
             int offset;
@@ -4665,12 +4641,12 @@
                 offset = 0;
             }
             INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            #line 4669 "Python/generated_cases.c.h"
+            #line 4645 "Python/generated_cases.c.h"
             DISPATCH();
         }
 
         TARGET(INSTRUMENTED_POP_JUMP_IF_NOT_NONE) {
-            #line 3374 "Python/bytecodes.c"
+            #line 3352 "Python/bytecodes.c"
             PyObject *value = POP();
             _Py_CODEUNIT *here = next_instr-1;
             int offset;
@@ -4683,30 +4659,30 @@
                  offset = oparg;
             }
             INSTRUMENTED_JUMP(here, next_instr + offset, PY_MONITORING_EVENT_BRANCH);
-            #line 4687 "Python/generated_cases.c.h"
+            #line 4663 "Python/generated_cases.c.h"
             DISPATCH();
         }
 
         TARGET(EXTENDED_ARG) {
-            #line 3389 "Python/bytecodes.c"
+            #line 3367 "Python/bytecodes.c"
             assert(oparg);
             opcode = next_instr->op.code;
             oparg = oparg << 8 | next_instr->op.arg;
             PRE_DISPATCH_GOTO();
             DISPATCH_GOTO();
-            #line 4698 "Python/generated_cases.c.h"
+            #line 4674 "Python/generated_cases.c.h"
         }
 
         TARGET(CACHE) {
-            #line 3397 "Python/bytecodes.c"
+            #line 3375 "Python/bytecodes.c"
             assert(0 && "Executing a cache.");
             Py_UNREACHABLE();
-            #line 4705 "Python/generated_cases.c.h"
+            #line 4681 "Python/generated_cases.c.h"
         }
 
         TARGET(RESERVED) {
-            #line 3402 "Python/bytecodes.c"
+            #line 3380 "Python/bytecodes.c"
             assert(0 && "Executing RESERVED instruction.");
             Py_UNREACHABLE();
-            #line 4712 "Python/generated_cases.c.h"
+            #line 4688 "Python/generated_cases.c.h"
         }

Python/instrumentation.c

@@ -14,7 +14,7 @@
 /* Uncomment this to dump debugging output when assertions fail */
 // #define INSTRUMENT_DEBUG 1
 
-static PyObject DISABLE =
+PyObject _PyInstrumentation_DISABLE =
 {
     .ob_refcnt = _Py_IMMORTAL_REFCNT,
     .ob_type = &PyBaseObject_Type
@@ -859,7 +859,7 @@ call_one_instrument(
         return -1;
     }
     Py_DECREF(res);
-    return (res == &DISABLE);
+    return (res == &_PyInstrumentation_DISABLE);
 }
 
 static const int8_t MOST_SIGNIFICANT_BITS[16] = {
@@ -1002,7 +1002,7 @@ _Py_call_instrumentation_2args(
     return call_instrumentation_vector(tstate, event, frame, instr, 4, args);
 }
 
-int
+_Py_CODEUNIT *
 _Py_call_instrumentation_jump(
     PyThreadState *tstate, int event,
     _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target)
@@ -1010,17 +1010,27 @@ _Py_call_instrumentation_jump(
     assert(event == PY_MONITORING_EVENT_JUMP ||
            event == PY_MONITORING_EVENT_BRANCH);
     assert(frame->prev_instr == instr);
+    /* Event should occur after the jump */
     frame->prev_instr = target;
     PyCodeObject *code = frame->f_code;
     int to = (int)(target - _PyCode_CODE(code));
     PyObject *to_obj = PyLong_FromLong(to * (int)sizeof(_Py_CODEUNIT));
     if (to_obj == NULL) {
-        return -1;
+        return NULL;
     }
     PyObject *args[4] = { NULL, NULL, NULL, to_obj };
     int err = call_instrumentation_vector(tstate, event, frame, instr, 3, args);
     Py_DECREF(to_obj);
-    return err;
+    if (err) {
+        return NULL;
+    }
+    if (frame->prev_instr != target) {
+        /* The callback has caused a jump (by setting the line number) */
+        return frame->prev_instr;
+    }
+    /* Reset prev_instr for INSTRUMENTED_LINE */
+    frame->prev_instr = instr;
+    return target;
 }
 
 static void
@@ -1076,13 +1086,14 @@ _Py_Instrumentation_GetLine(PyCodeObject *code, int index)
 }
 
 int
-_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr)
+_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev)
 {
     frame->prev_instr = instr;
     PyCodeObject *code = frame->f_code;
     assert(is_version_up_to_date(code, tstate->interp));
     assert(instrumentation_cross_checks(tstate->interp, code));
     int i = (int)(instr - _PyCode_CODE(code));
+
     _PyCoMonitoringData *monitoring = code->_co_monitoring;
     _PyCoLineInstrumentationData *line_data = &monitoring->lines[i];
     uint8_t original_opcode = line_data->original_opcode;
@@ -1092,6 +1103,18 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
     PyInterpreterState *interp = tstate->interp;
     int8_t line_delta = line_data->line_delta;
     int line = compute_line(code, i, line_delta);
+    assert(line >= 0);
+    int prev_index = (int)(prev - _PyCode_CODE(code));
+    int prev_line = _Py_Instrumentation_GetLine(code, prev_index);
+    if (prev_line == line) {
+        int prev_opcode = _PyCode_CODE(code)[prev_index].op.code;
+        /* RESUME and INSTRUMENTED_RESUME are needed for the operation of
+         * instrumentation, so must never be hidden by an INSTRUMENTED_LINE.
+         */
+        if (prev_opcode != RESUME && prev_opcode != INSTRUMENTED_RESUME) {
+            goto done;
+        }
+    }
     uint8_t tools = code->_co_monitoring->line_tools != NULL ?
         code->_co_monitoring->line_tools[i] :
         (interp->monitors.tools[PY_MONITORING_EVENT_LINE] |
@@ -1275,30 +1298,92 @@ initialize_lines(PyCodeObject *code)
                 line_data[i].original_opcode = 0;
                 break;
             default:
+                /* Set original_opcode to the opcode iff the instruction
+                 * starts a line, and thus should be instrumented.
+                 * This saves having to perform this check every time the
+                 * we turn instrumentation on or off, and serves as a sanity
+                 * check when debugging.
+                 */
                 if (line != current_line && line >= 0) {
                     line_data[i].original_opcode = opcode;
                 }
                 else {
                     line_data[i].original_opcode = 0;
                 }
-                if (line >= 0) {
+                current_line = line;
-                    current_line = line;
-                }
         }
         for (int j = 1; j < length; j++) {
             line_data[i+j].original_opcode = 0;
             line_data[i+j].line_delta = NO_LINE;
         }
-        switch (opcode) {
-            case RETURN_VALUE:
-            case RAISE_VARARGS:
-            case RERAISE:
-                /* Blocks of code after these terminators
-                 * should be treated as different lines */
-                current_line = -1;
-        }
         i += length;
     }
+    for (int i = code->_co_firsttraceable; i < code_len; ) {
+        int opcode = _Py_GetBaseOpcode(code, i);
+        int oparg = 0;
+        while (opcode == EXTENDED_ARG) {
+            oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg;
+            i++;
+            opcode = _Py_GetBaseOpcode(code, i);
+        }
+        oparg = (oparg << 8) | _PyCode_CODE(code)[i].op.arg;
+        i += instruction_length(code, i);
+        int target = -1;
+        switch (opcode) {
+            case POP_JUMP_IF_FALSE:
+            case POP_JUMP_IF_TRUE:
+            case POP_JUMP_IF_NONE:
+            case POP_JUMP_IF_NOT_NONE:
+            case JUMP_FORWARD:
+            {
+                target = i + oparg;
+                break;
+            }
+            case FOR_ITER:
+            case SEND:
+            {
+                /* Skip over END_FOR/END_SEND */
+                target = i + oparg + 1;
+                break;
+            }
+            case JUMP_BACKWARD:
+            case JUMP_BACKWARD_NO_INTERRUPT:
+            {
+                target = i - oparg;
+                break;
+            }
+            default:
+                continue;
+        }
+        assert(target >= 0);
+        if (line_data[target].line_delta != NO_LINE) {
+            line_data[target].original_opcode = _Py_GetBaseOpcode(code, target);
+        }
+    }
+    /* Scan exception table */
+    unsigned char *start = (unsigned char *)PyBytes_AS_STRING(code->co_exceptiontable);
+    unsigned char *end = start + PyBytes_GET_SIZE(code->co_exceptiontable);
+    unsigned char *scan = start;
+    while (scan < end) {
+        int start_offset, size, handler;
+        scan = parse_varint(scan, &start_offset);
+        assert(start_offset >= 0 && start_offset < code_len);
+        scan = parse_varint(scan, &size);
+        assert(size >= 0 && start_offset+size <= code_len);
+        scan = parse_varint(scan, &handler);
+        assert(handler >= 0 && handler < code_len);
+        int depth_and_lasti;
+        scan = parse_varint(scan, &depth_and_lasti);
+        int original_opcode = _Py_GetBaseOpcode(code, handler);
+        /* Skip if not the start of a line.
+         * END_ASYNC_FOR is a bit special as it marks the end of
+         * an `async for` loop, which should not generate its own
+         * line event. */
+        if (line_data[handler].line_delta != NO_LINE &&
+            original_opcode != END_ASYNC_FOR) {
+            line_data[handler].original_opcode = original_opcode;
+        }
+    }
 }
 
 static void
@@ -2010,7 +2095,7 @@ PyObject *_Py_CreateMonitoringObject(void)
     if (mod == NULL) {
         return NULL;
     }
-    if (PyObject_SetAttrString(mod, "DISABLE", &DISABLE)) {
+    if (PyObject_SetAttrString(mod, "DISABLE", &_PyInstrumentation_DISABLE)) {
         goto error;
     }
     if (PyObject_SetAttrString(mod, "MISSING", &_PyInstrumentation_MISSING)) {

Python/legacy_tracing.c

@@ -4,6 +4,7 @@
 
 #include <stddef.h>
 #include "Python.h"
+#include "opcode.h"
 #include "pycore_ceval.h"
 #include "pycore_object.h"
 #include "pycore_sysmodule.h"
@@ -213,7 +214,6 @@ trace_line(
     if (line < 0) {
         Py_RETURN_NONE;
     }
-    frame ->f_last_traced_line = line;
     Py_INCREF(frame);
     frame->f_lineno = line;
     int err = tstate->c_tracefunc(tstate->c_traceobj, frame, self->event, Py_None);
@@ -245,14 +245,12 @@ sys_trace_line_func(
         return NULL;
     }
     assert(args[0] == (PyObject *)frame->f_frame->f_code);
-    if (frame ->f_last_traced_line == line) {
-        /* Already traced this line */
-        Py_RETURN_NONE;
-    }
     return trace_line(tstate, self, frame, line);
 }
 
-
+/* sys.settrace generates line events for all backward
+ * edges, even if on the same line.
+ * Handle that case here */
 static PyObject *
 sys_trace_jump_func(
     _PyLegacyEventHandler *self, PyObject *const *args,
@@ -268,61 +266,33 @@ sys_trace_jump_func(
     assert(from >= 0);
     int to = _PyLong_AsInt(args[2])/sizeof(_Py_CODEUNIT);
     assert(to >= 0);
+    if (to > from) {
+        /* Forward jump */
+        return &_PyInstrumentation_DISABLE;
+    }
+    PyCodeObject *code = (PyCodeObject *)args[0];
+    assert(PyCode_Check(code));
+    /* We can call _Py_Instrumentation_GetLine because we always set
+    * line events for tracing */
+    int to_line = _Py_Instrumentation_GetLine(code, to);
+    int from_line = _Py_Instrumentation_GetLine(code, from);
+    if (to_line != from_line) {
+        /* Will be handled by target INSTRUMENTED_LINE */
+        return &_PyInstrumentation_DISABLE;
+    }
     PyFrameObject *frame = PyEval_GetFrame();
     if (frame == NULL) {
         PyErr_SetString(PyExc_SystemError,
                         "Missing frame when calling trace function.");
         return NULL;
     }
-    if (!frame->f_trace_lines) {
-        Py_RETURN_NONE;
-    }
-    PyCodeObject *code = (PyCodeObject *)args[0];
-    assert(PyCode_Check(code));
     assert(code == frame->f_frame->f_code);
-    /* We can call _Py_Instrumentation_GetLine because we always set
+    if (!frame->f_trace_lines) {
-    * line events for tracing */
-    int to_line = _Py_Instrumentation_GetLine(code, to);
-    /* Backward jump: Always generate event
-     * Forward jump: Only generate event if jumping to different line. */
-    if (to > from && frame->f_last_traced_line == to_line) {
-        /* Already traced this line */
         Py_RETURN_NONE;
     }
     return trace_line(tstate, self, frame, to_line);
 }
 
-/* We don't care about the exception here,
- * we just treat it as a possible new line
- */
-static PyObject *
-sys_trace_exception_handled(
-    _PyLegacyEventHandler *self, PyObject *const *args,
-    size_t nargsf, PyObject *kwnames
-) {
-    assert(kwnames == NULL);
-    PyThreadState *tstate = _PyThreadState_GET();
-    if (tstate->c_tracefunc == NULL) {
-        Py_RETURN_NONE;
-    }
-    assert(PyVectorcall_NARGS(nargsf) == 3);
-    PyFrameObject *frame = PyEval_GetFrame();
-    PyCodeObject *code = (PyCodeObject *)args[0];
-    assert(PyCode_Check(code));
-    assert(code == frame->f_frame->f_code);
-    assert(PyLong_Check(args[1]));
-    int offset = _PyLong_AsInt(args[1])/sizeof(_Py_CODEUNIT);
-    /* We can call _Py_Instrumentation_GetLine because we always set
-    * line events for tracing */
-    int line = _Py_Instrumentation_GetLine(code, offset);
-    if (frame->f_last_traced_line == line) {
-        /* Already traced this line */
-        Py_RETURN_NONE;
-    }
-    return trace_line(tstate, self, frame, line);
-}
-
-
 PyTypeObject _PyLegacyEventHandler_Type = {
     PyVarObject_HEAD_INIT(&PyType_Type, 0)
     "sys.legacy_event_handler",
@@ -487,7 +457,7 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
         }
         if (set_callbacks(PY_MONITORING_SYS_TRACE_ID,
             (vectorcallfunc)sys_trace_jump_func, PyTrace_LINE,
-                        PY_MONITORING_EVENT_JUMP, PY_MONITORING_EVENT_BRANCH)) {
+                        PY_MONITORING_EVENT_JUMP, -1)) {
             return -1;
         }
         if (set_callbacks(PY_MONITORING_SYS_TRACE_ID,
@@ -495,11 +465,6 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg)
                         PY_MONITORING_EVENT_INSTRUCTION, -1)) {
             return -1;
         }
-        if (set_callbacks(PY_MONITORING_SYS_TRACE_ID,
-            (vectorcallfunc)sys_trace_exception_handled, PyTrace_LINE,
-                        PY_MONITORING_EVENT_EXCEPTION_HANDLED, -1)) {
-            return -1;
-        }
     }
 
     int delta = (func != NULL) - (tstate->c_tracefunc != NULL);

Python/opcode_metadata.h

@@ -367,8 +367,6 @@ _PyOpcode_num_popped(int opcode, int oparg, bool jump) {
             return 2;
         case SWAP:
             return (oparg-2) + 2;
-        case INSTRUMENTED_LINE:
-            return 0;
         case INSTRUMENTED_INSTRUCTION:
             return 0;
         case INSTRUMENTED_JUMP_FORWARD:
@@ -759,8 +757,6 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) {
             return 1;
         case SWAP:
             return (oparg-2) + 2;
-        case INSTRUMENTED_LINE:
-            return 0;
         case INSTRUMENTED_INSTRUCTION:
             return 0;
         case INSTRUMENTED_JUMP_FORWARD:
@@ -976,7 +972,6 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = {
     [COPY] = { true, INSTR_FMT_IB },
     [BINARY_OP] = { true, INSTR_FMT_IBC },
     [SWAP] = { true, INSTR_FMT_IB },
-    [INSTRUMENTED_LINE] = { true, INSTR_FMT_IX },
     [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX },
     [INSTRUMENTED_JUMP_FORWARD] = { true, INSTR_FMT_IB },
     [INSTRUMENTED_JUMP_BACKWARD] = { true, INSTR_FMT_IB },

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -301,7 +301,7 @@ Objects/object.c	-	_Py_NotImplementedStruct	-
 Objects/setobject.c	-	_dummy_struct	-
 Objects/setobject.c	-	_PySet_Dummy	-
 Objects/sliceobject.c	-	_Py_EllipsisObject	-
-Python/instrumentation.c	-	DISABLE	-
+Python/instrumentation.c	-	_PyInstrumentation_DISABLE	-
 Python/instrumentation.c	-	_PyInstrumentation_MISSING	-