bpo-45753: Interpreter internal tweaks (GH-29575)

* Split exit paths into exceptional and non-exceptional. * Move exit tracing code to individual bytecodes. * Wrap all trace entry and exit events in macros to make them clearer and easier to enhance. * Move return sequence into RETURN_VALUE, YIELD_VALUE and YIELD_FROM. Distinguish between normal trace events and dtrace events.
2025-08-04 00:48:58 +00:00 · 2021-12-01 12:09:36 +00:00 · 2021-12-01 12:09:36 +00:00 · 49444fb807
commit 49444fb807
parent 0aa0bd0563
3 changed files with 214 additions and 153 deletions
--- a/Python/ceval.c
+++ b/Python/ceval.c
@ -102,8 +102,8 @@ static InterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
                        PyObject *locals, PyObject* const* args,
                        size_t argcount, PyObject *kwnames);
-static int
-_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame);
+static void
+_PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame *frame);

 #define NAME_ERROR_MSG \
    "name '%.200s' is not defined"
@ -1390,7 +1390,7 @@ eval_frame_handle_pending(PyThreadState *tstate)

 /* Get opcode and oparg from original instructions, not quickened form. */
 #define TRACING_NEXTOPARG() do { \
-        _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(co->co_code))[INSTR_OFFSET()]; \
+        _Py_CODEUNIT word = ((_Py_CODEUNIT *)PyBytes_AS_STRING(frame->f_code->co_code))[INSTR_OFFSET()]; \
        opcode = _Py_OPCODE(word); \
        oparg = _Py_OPARG(word); \
    } while (0)
@ -1462,20 +1462,20 @@ eval_frame_handle_pending(PyThreadState *tstate)
 #ifdef LLTRACE
 #define PUSH(v)         { (void)(BASIC_PUSH(v), \
                          lltrace && prtrace(tstate, TOP(), "push")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); }
+                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); }
 #define POP()           ((void)(lltrace && prtrace(tstate, TOP(), "pop")), \
                         BASIC_POP())
 #define STACK_GROW(n)   do { \
                          assert(n >= 0); \
                          (void)(BASIC_STACKADJ(n), \
                          lltrace && prtrace(tstate, TOP(), "stackadj")); \
-                          assert(STACK_LEVEL() <= co->co_stacksize); \
+                          assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                        } while (0)
 #define STACK_SHRINK(n) do { \
                            assert(n >= 0); \
                            (void)(lltrace && prtrace(tstate, TOP(), "stackadj")); \
                            (void)(BASIC_STACKADJ(-(n))); \
-                            assert(STACK_LEVEL() <= co->co_stacksize); \
+                            assert(STACK_LEVEL() <= frame->f_code->co_stacksize); \
                        } while (0)
 #define EXT_POP(STACK_POINTER) ((void)(lltrace && \
                                prtrace(tstate, (STACK_POINTER)[-1], "ext_pop")), \
@ -1537,6 +1537,40 @@ eval_frame_handle_pending(PyThreadState *tstate)
    STAT_INC(LOAD_##attr_or_method, hit); \
    Py_INCREF(res);

+#define TRACE_FUNCTION_EXIT() \
+    if (cframe.use_tracing) { \
+        if (trace_function_exit(tstate, frame, retval)) { \
+            Py_DECREF(retval); \
+            goto exit_unwind; \
+        } \
+    }
+
+#define DTRACE_FUNCTION_EXIT() \
+    if (PyDTrace_FUNCTION_RETURN_ENABLED()) { \
+        dtrace_function_return(frame); \
+    }
+
+#define TRACE_FUNCTION_UNWIND()  \
+    if (cframe.use_tracing) { \
+        /* Since we are already unwinding, \
+         * we dont't care if this raises */ \
+        trace_function_exit(tstate, frame, NULL); \
+    }
+
+#define TRACE_FUNCTION_ENTRY() \
+    if (cframe.use_tracing) { \
+        if (trace_function_entry(tstate, frame)) { \
+            goto exit_unwind; \
+        } \
+    }
+
+#define DTRACE_FUNCTION_ENTRY()  \
+    if (PyDTrace_FUNCTION_ENTRY_ENABLED()) { \
+        dtrace_function_entry(frame); \
+    }
+
+
+
 static int
 trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
 {
@ -1576,6 +1610,24 @@ trace_function_entry(PyThreadState *tstate, InterpreterFrame *frame)
    return 0;
 }

+static int
+trace_function_exit(PyThreadState *tstate, InterpreterFrame *frame, PyObject *retval)
+{
+    if (tstate->c_tracefunc) {
+        if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
+                                    tstate, frame, PyTrace_RETURN, retval)) {
+            return -1;
+        }
+    }
+    if (tstate->c_profilefunc) {
+        if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
+                                    tstate, frame, PyTrace_RETURN, retval)) {
+            return -1;
+        }
+    }
+    return 0;
+}
+
 static PyObject *
 make_coro(PyThreadState *tstate, PyFunctionObject *func,
          PyObject *locals,
@ -1583,7 +1635,8 @@ make_coro(PyThreadState *tstate, PyFunctionObject *func,
          PyObject *kwnames);

 static int
-skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
+skip_backwards_over_extended_args(PyCodeObject *code, int offset)
+{
    _Py_CODEUNIT *instrs = (_Py_CODEUNIT *)PyBytes_AS_STRING(code->co_code);
    while (offset > 0 && _Py_OPCODE(instrs[offset-1]) == EXTENDED_ARG) {
        offset--;
@ -1591,6 +1644,14 @@ skip_backwards_over_extended_args(PyCodeObject *code, int offset) {
    return offset;
 }

+static InterpreterFrame *
+pop_frame(PyThreadState *tstate, InterpreterFrame *frame)
+{
+    InterpreterFrame *prev_frame = frame->previous;
+    _PyEvalFrameClearAndPop(tstate, frame);
+    return prev_frame;
+}
+
 PyObject* _Py_HOT_FUNCTION
 _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int throwflag)
 {
@ -1606,7 +1667,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
 #endif
    int opcode;        /* Current opcode */
    int oparg;         /* Current opcode argument, if any */
-    PyObject *retval = NULL;            /* Return value */
    _Py_atomic_int * const eval_breaker = &tstate->interp->ceval.eval_breaker;

    CFrame cframe;
@ -1625,82 +1685,77 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, InterpreterFrame *frame, int thr
    frame->previous = prev_cframe->current_frame;
    cframe.current_frame = frame;

+    /* support for generator.throw() */
+    if (throwflag) {
+        if (_Py_EnterRecursiveCall(tstate, "")) {
+            tstate->recursion_remaining--;
+            goto exit_unwind;
+        }
+        TRACE_FUNCTION_ENTRY();
+        DTRACE_FUNCTION_ENTRY();
+        goto resume_with_error;
+    }
+
+    /* Local "register" variables.
+     * These are cached values from the frame and code object.  */
+
+    PyObject *names;
+    PyObject *consts;
+    _Py_CODEUNIT *first_instr;
+    _Py_CODEUNIT *next_instr;
+    PyObject **stack_pointer;
+
+/* Sets the above local variables from the frame */
+#define SET_LOCALS_FROM_FRAME() \
+    { \
+        PyCodeObject *co = frame->f_code; \
+        names = co->co_names; \
+        consts = co->co_consts; \
+        first_instr = co->co_firstinstr; \
+    } \
+    assert(frame->f_lasti >= -1); \
+    next_instr = first_instr + frame->f_lasti + 1; \
+    stack_pointer = _PyFrame_GetStackPointer(frame); \
+    /* Set stackdepth to -1. \
+        Update when returning or calling trace function. \
+        Having stackdepth <= 0 ensures that invalid \
+        values are not visible to the cycle GC. \
+        We choose -1 rather than 0 to assist debugging. \
+        */ \
+    frame->stacktop = -1;
+
+
 start_frame:
    if (_Py_EnterRecursiveCall(tstate, "")) {
        tstate->recursion_remaining--;
-        goto exit_eval_frame;
+        goto exit_unwind;
    }

    assert(tstate->cframe == &cframe);
    assert(frame == cframe.current_frame);

-    if (cframe.use_tracing) {
-        if (trace_function_entry(tstate, frame)) {
-            goto exit_eval_frame;
-        }
-    }
+    TRACE_FUNCTION_ENTRY();
+    DTRACE_FUNCTION_ENTRY();

-    if (PyDTrace_FUNCTION_ENTRY_ENABLED())
-        dtrace_function_entry(frame);
-
-    PyCodeObject *co = frame->f_code;
-    /* Increment the warmup counter and quicken if warm enough
-     * _Py_Quicken is idempotent so we don't worry about overflow */
-    if (!PyCodeObject_IsWarmedUp(co)) {
-        PyCodeObject_IncrementWarmup(co);
-        if (PyCodeObject_IsWarmedUp(co)) {
-            if (_Py_Quicken(co)) {
-                goto exit_eval_frame;
-            }
-        }
+    if (_Py_IncrementCountAndMaybeQuicken(frame->f_code) < 0) {
+        goto exit_unwind;
    }
+    frame->f_state = FRAME_EXECUTING;

 resume_frame:
-    co = frame->f_code;
-    PyObject *names = co->co_names;
-    PyObject *consts = co->co_consts;
-    _Py_CODEUNIT *first_instr = co->co_firstinstr;
-    /*
-       frame->f_lasti refers to the index of the last instruction,
-       unless it's -1 in which case next_instr should be first_instr.
-
-       YIELD_FROM sets frame->f_lasti to itself, in order to repeatedly yield
-       multiple values.
-
-       When the PREDICT() macros are enabled, some opcode pairs follow in
-       direct succession. A successful prediction effectively links the two
-       codes together as if they were a single new opcode, but the value
-       of frame->f_lasti is correctly updated so potential inlined calls
-       or lookups of frame->f_lasti are aways correct when the macros are used.
-    */
-    assert(frame->f_lasti >= -1);
-    _Py_CODEUNIT *next_instr = first_instr + frame->f_lasti + 1;
-    PyObject **stack_pointer = _PyFrame_GetStackPointer(frame);
-    /* Set stackdepth to -1.
-     * Update when returning or calling trace function.
-       Having stackdepth <= 0 ensures that invalid
-       values are not visible to the cycle GC.
-       We choose -1 rather than 0 to assist debugging.
-     */
-    frame->stacktop = -1;
-    frame->f_state = FRAME_EXECUTING;
+    SET_LOCALS_FROM_FRAME();

 #ifdef LLTRACE
    _Py_IDENTIFIER(__ltrace__);
    {
        int r = _PyDict_ContainsId(GLOBALS(), &PyId___ltrace__);
        if (r < 0) {
-            goto exit_eval_frame;
+            goto exit_unwind;
        }
        lltrace = r;
    }
 #endif

-    if (throwflag) { /* support for generator.throw() */
-        throwflag = 0;
-        goto error;
-    }
-
 #ifdef Py_DEBUG
    /* _PyEval_EvalFrameDefault() must not be called with an exception set,
       because it can clear it (directly or indirectly) and so the
@ -1711,7 +1766,7 @@ resume_frame:
 check_eval_breaker:
    {
        assert(STACK_LEVEL() >= 0); /* else underflow */
-        assert(STACK_LEVEL() <= co->co_stacksize);  /* else overflow */
+        assert(STACK_LEVEL() <= frame->f_code->co_stacksize);  /* else overflow */
        assert(!_PyErr_Occurred(tstate));

        /* Do periodic things.  Doing this every time through
@ -2418,11 +2473,24 @@ check_eval_breaker:
        }

        TARGET(RETURN_VALUE) {
-            retval = POP();
+            PyObject *retval = POP();
            assert(EMPTY());
            frame->f_state = FRAME_RETURNED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            if (frame->depth) {
+                frame = cframe.current_frame = pop_frame(tstate, frame);
+                _PyFrame_StackPush(frame, retval);
+                goto resume_frame;
+            }
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
        }

        TARGET(GET_AITER) {
@ -2562,9 +2630,11 @@ check_eval_breaker:
        }

        TARGET(YIELD_FROM) {
+            assert(frame->depth == 0);
            PyObject *v = POP();
            PyObject *receiver = TOP();
            PySendResult gen_status;
+            PyObject *retval;
            if (tstate->c_tracefunc == NULL) {
                gen_status = PyIter_Send(receiver, v, &retval);
            } else {
@ -2610,13 +2680,22 @@ check_eval_breaker:
            frame->f_lasti -= 1;
            frame->f_state = FRAME_SUSPENDED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
        }

        TARGET(YIELD_VALUE) {
-            retval = POP();
+            assert(frame->depth == 0);
+            PyObject *retval = POP();

-            if (co->co_flags & CO_ASYNC_GENERATOR) {
+            if (frame->f_code->co_flags & CO_ASYNC_GENERATOR) {
                PyObject *w = _PyAsyncGenValueWrapperNew(retval);
                Py_DECREF(retval);
                if (w == NULL) {
@ -2627,7 +2706,15 @@ check_eval_breaker:
            }
            frame->f_state = FRAME_SUSPENDED;
            _PyFrame_SetStackPointer(frame, stack_pointer);
-            goto exiting;
+            TRACE_FUNCTION_EXIT();
+            DTRACE_FUNCTION_EXIT();
+            _Py_LeaveRecursiveCall(tstate);
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return retval;
        }

        TARGET(GEN_START) {
@ -3100,7 +3187,7 @@ check_eval_breaker:
            format_exc_check_arg(
                tstate, PyExc_UnboundLocalError,
                UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_localsplusnames, oparg)
+                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
                );
            goto error;
        }
@ -3125,15 +3212,15 @@ check_eval_breaker:
                Py_DECREF(oldobj);
                DISPATCH();
            }
-            format_exc_unbound(tstate, co, oparg);
+            format_exc_unbound(tstate, frame->f_code, oparg);
            goto error;
        }

        TARGET(LOAD_CLASSDEREF) {
            PyObject *name, *value, *locals = LOCALS();
            assert(locals);
-            assert(oparg >= 0 && oparg < co->co_nlocalsplus);
-            name = PyTuple_GET_ITEM(co->co_localsplusnames, oparg);
+            assert(oparg >= 0 && oparg < frame->f_code->co_nlocalsplus);
+            name = PyTuple_GET_ITEM(frame->f_code->co_localsplusnames, oparg);
            if (PyDict_CheckExact(locals)) {
                value = PyDict_GetItemWithError(locals, name);
                if (value != NULL) {
@ -3156,7 +3243,7 @@ check_eval_breaker:
                PyObject *cell = GETLOCAL(oparg);
                value = PyCell_GET(cell);
                if (value == NULL) {
-                    format_exc_unbound(tstate, co, oparg);
+                    format_exc_unbound(tstate, frame->f_code, oparg);
                    goto error;
                }
                Py_INCREF(value);
@ -3169,7 +3256,7 @@ check_eval_breaker:
            PyObject *cell = GETLOCAL(oparg);
            PyObject *value = PyCell_GET(cell);
            if (value == NULL) {
-                format_exc_unbound(tstate, co, oparg);
+                format_exc_unbound(tstate, frame->f_code, oparg);
                goto error;
            }
            Py_INCREF(value);
@ -3918,18 +4005,15 @@ check_eval_breaker:
        TARGET(JUMP_ABSOLUTE) {
            PREDICTED(JUMP_ABSOLUTE);
            assert(oparg < INSTR_OFFSET());
-            /* Increment the warmup counter and quicken if warm enough
-            * _Py_Quicken is idempotent so we don't worry about overflow */
-            if (!PyCodeObject_IsWarmedUp(co)) {
-                PyCodeObject_IncrementWarmup(co);
-                if (PyCodeObject_IsWarmedUp(co)) {
-                    if (_Py_Quicken(co)) {
-                        goto error;
-                    }
-                    int nexti = INSTR_OFFSET();
-                    first_instr = co->co_firstinstr;
-                    next_instr = first_instr + nexti;
+            int err = _Py_IncrementCountAndMaybeQuicken(frame->f_code);
+            if (err) {
+                if (err < 0) {
+                    goto error;
                }
+                /* Update first_instr and next_instr to point to newly quickened code */
+                int nexti = INSTR_OFFSET();
+                first_instr = frame->f_code->co_firstinstr;
+                next_instr = first_instr + nexti;
            }
            JUMPTO(oparg);
            CHECK_EVAL_BREAKER();
@ -4036,7 +4120,7 @@ check_eval_breaker:
            PyObject *iter;
            if (PyCoro_CheckExact(iterable)) {
                /* `iterable` is a coroutine */
-                if (!(co->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
+                if (!(frame->f_code->co_flags & (CO_COROUTINE | CO_ITERABLE_COROUTINE))) {
                    /* and it is used in a 'yield from' expression of a
                       regular generator. */
                    Py_DECREF(iterable);
@ -4906,7 +4990,7 @@ check_eval_breaker:
 #else
        case DO_TRACING: {
 #endif
-            int instr_prev = skip_backwards_over_extended_args(co, frame->f_lasti);
+            int instr_prev = skip_backwards_over_extended_args(frame->f_code, frame->f_lasti);
            frame->f_lasti = INSTR_OFFSET();
            TRACING_NEXTOPARG();
            if (PyDTrace_LINE_ENABLED()) {
@ -5016,7 +5100,7 @@ unbound_local_error:
        {
            format_exc_check_arg(tstate, PyExc_UnboundLocalError,
                UNBOUNDLOCAL_ERROR_MSG,
-                PyTuple_GetItem(co->co_localsplusnames, oparg)
+                PyTuple_GetItem(frame->f_code->co_localsplusnames, oparg)
            );
            goto error;
        }
@ -5039,8 +5123,7 @@ error:
        }

        if (tstate->c_tracefunc != NULL) {
-            /* Make sure state is set to FRAME_EXECUTING for tracing */
-            assert(frame->f_state == FRAME_EXECUTING);
+            /* Make sure state is set to FRAME_UNWINDING for tracing */
            frame->f_state = FRAME_UNWINDING;
            call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj,
                           tstate, frame);
@ -5051,9 +5134,8 @@ exception_unwind:
        /* We can't use frame->f_lasti here, as RERAISE may have set it */
        int offset = INSTR_OFFSET()-1;
        int level, handler, lasti;
-        if (get_exception_handler(co, offset, &level, &handler, &lasti) == 0) {
+        if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) {
            // No handlers, so exit.
-            assert(retval == NULL);
            assert(_PyErr_Occurred(tstate));

            /* Pop remaining stack entries. */
@ -5065,7 +5147,9 @@ exception_unwind:
            assert(STACK_LEVEL() == 0);
            _PyFrame_SetStackPointer(frame, stack_pointer);
            frame->f_state = FRAME_RAISED;
-            goto exiting;
+            TRACE_FUNCTION_UNWIND();
+            DTRACE_FUNCTION_EXIT();
+            goto exit_unwind;
        }

        assert(STACK_LEVEL() >= level);
@ -5106,48 +5190,22 @@ exception_unwind:
        DISPATCH();
    }

-exiting:
-    if (cframe.use_tracing) {
-        if (tstate->c_tracefunc) {
-            if (call_trace_protected(tstate->c_tracefunc, tstate->c_traceobj,
-                                     tstate, frame, PyTrace_RETURN, retval)) {
-                Py_CLEAR(retval);
-            }
-        }
-        if (tstate->c_profilefunc) {
-            if (call_trace_protected(tstate->c_profilefunc, tstate->c_profileobj,
-                                     tstate, frame, PyTrace_RETURN, retval)) {
-                Py_CLEAR(retval);
-            }
-        }
-    }
-
-    /* pop frame */
-exit_eval_frame:
-    if (PyDTrace_FUNCTION_RETURN_ENABLED())
-        dtrace_function_return(frame);
+exit_unwind:
+    assert(_PyErr_Occurred(tstate));
    _Py_LeaveRecursiveCall(tstate);
-
-    if (frame->depth) {
-        cframe.current_frame = frame->previous;
-        _PyFrame_StackPush(cframe.current_frame, retval);
-        if (_PyEvalFrameClearAndPop(tstate, frame)) {
-            retval = NULL;
-        }
-        frame = cframe.current_frame;
-        if (retval == NULL) {
-            assert(_PyErr_Occurred(tstate));
-            throwflag = 1;
-        }
-        retval = NULL;
-        goto resume_frame;
+    if (frame->depth == 0) {
+        /* Restore previous cframe and exit */
+        tstate->cframe = cframe.previous;
+        tstate->cframe->use_tracing = cframe.use_tracing;
+        assert(tstate->cframe->current_frame == frame->previous);
+        return NULL;
    }
+    frame = cframe.current_frame = pop_frame(tstate, frame);
+
+resume_with_error:
+    SET_LOCALS_FROM_FRAME();
+    goto error;

-    /* Restore previous cframe. */
-    tstate->cframe = cframe.previous;
-    tstate->cframe->use_tracing = cframe.use_tracing;
-    assert(tstate->cframe->current_frame == frame->previous);
-    return _Py_CheckFunctionResult(tstate, NULL, retval, __func__);
 }

 static void
@ -5770,15 +5828,14 @@ fail:
    return NULL;
 }

-static int
+static void
 _PyEvalFrameClearAndPop(PyThreadState *tstate, InterpreterFrame * frame)
 {
-    --tstate->recursion_remaining;
+    tstate->recursion_remaining--;
    assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0);
    _PyFrame_Clear(frame);
-    ++tstate->recursion_remaining;
+    tstate->recursion_remaining++;
    _PyThreadState_PopFrame(tstate, frame);
-    return 0;
 }

 PyObject *
@ -5812,9 +5869,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
    }
    PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
    assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame));
-    if (_PyEvalFrameClearAndPop(tstate, frame)) {
-        retval = NULL;
-    }
+    _PyEvalFrameClearAndPop(tstate, frame);
    return retval;
 }