bpo-46409: Make generators in bytecode (GH-30633)

* Add RETURN_GENERATOR and JUMP_NO_INTERRUPT opcodes.

* Trim frame and generator by word each.

* Minor refactor of frame.c

* Update test.test_sys to account for smaller frames.

* Treat generator functions as normal functions when evaluating and specializing.

Doc/library/dis.rst

@@ -942,6 +942,13 @@ All of the following opcodes use their arguments.
    Set bytecode counter to *target*.
 
 
+.. opcode:: JUMP_NO_INTERRUPT (target)
+
+   Set bytecode counter to *target*. Do not check for interrupts.
+
+   .. versionadded:: 3.11
+
+
 .. opcode:: FOR_ITER (delta)
 
    TOS is an :term:`iterator`.  Call its :meth:`~iterator.__next__` method.  If
@@ -1220,6 +1227,14 @@ All of the following opcodes use their arguments.
    .. versionadded:: 3.11
 
 
+.. opcode:: RETURN_GENERATOR
+
+    Create a generator, coroutine, or async generator from the current frame.
+    Clear the current frame and return the newly created generator.
+
+    .. versionadded:: 3.11
+
+
 .. opcode:: HAVE_ARGUMENT
 
    This is not really an opcode.  It identifies the dividing line between

Include/cpython/genobject.h

@@ -13,7 +13,6 @@ extern "C" {
    and coroutine objects. */
 #define _PyGenObject_HEAD(prefix)                                           \
     PyObject_HEAD                                                           \
-    /* Note: gi_frame can be NULL if the generator is "finished" */         \
     /* The code object backing the generator */                             \
     PyCodeObject *prefix##_code;                                            \
     /* List of weak reference. */                                           \

Include/internal/pycore_frame.h

@@ -41,12 +41,12 @@ typedef struct _interpreter_frame {
     PyObject *f_locals; /* Strong reference, may be NULL */
     PyCodeObject *f_code; /* Strong reference */
     PyFrameObject *frame_obj; /* Strong reference, may be NULL */
-    PyObject *generator; /* Borrowed reference, may be NULL */
     struct _interpreter_frame *previous;
     int f_lasti;       /* Last instruction if called */
     int stacktop;     /* Offset of TOS from localsplus  */
     PyFrameState f_state;  /* What state the frame is in */
     bool is_entry;  // Whether this is the "root" frame for the current CFrame.
+    bool is_generator;
     PyObject *localsplus[1];
 } InterpreterFrame;
 
@@ -100,10 +100,10 @@ _PyFrame_InitializeSpecials(
     frame->f_locals = Py_XNewRef(locals);
     frame->stacktop = nlocalsplus;
     frame->frame_obj = NULL;
-    frame->generator = NULL;
     frame->f_lasti = -1;
     frame->f_state = FRAME_CREATED;
     frame->is_entry = false;
+    frame->is_generator = false;
 }
 
 /* Gets the pointer to the locals array

Include/opcode.h

@@ -38,6 +38,7 @@ extern "C" {
 #define LOAD_BUILD_CLASS                 71
 #define GET_AWAITABLE                    73
 #define LOAD_ASSERTION_ERROR             74
+#define RETURN_GENERATOR                 75
 #define LIST_TO_TUPLE                    82
 #define RETURN_VALUE                     83
 #define IMPORT_STAR                      84
@@ -89,6 +90,7 @@ extern "C" {
 #define RAISE_VARARGS                   130
 #define MAKE_FUNCTION                   132
 #define BUILD_SLICE                     133
+#define JUMP_NO_INTERRUPT               134
 #define MAKE_CELL                       135
 #define LOAD_CLOSURE                    136
 #define LOAD_DEREF                      137
@@ -157,18 +159,18 @@ extern "C" {
 #define LOAD_GLOBAL_BUILTIN              66
 #define LOAD_METHOD_ADAPTIVE             67
 #define LOAD_METHOD_CACHED               72
-#define LOAD_METHOD_CLASS                75
+#define LOAD_METHOD_CLASS                76
-#define LOAD_METHOD_MODULE               76
+#define LOAD_METHOD_MODULE               77
-#define LOAD_METHOD_NO_DICT              77
+#define LOAD_METHOD_NO_DICT              78
-#define STORE_ATTR_ADAPTIVE              78
+#define STORE_ATTR_ADAPTIVE              79
-#define STORE_ATTR_INSTANCE_VALUE        79
+#define STORE_ATTR_INSTANCE_VALUE        80
-#define STORE_ATTR_SLOT                  80
+#define STORE_ATTR_SLOT                  81
-#define STORE_ATTR_WITH_HINT             81
+#define STORE_ATTR_WITH_HINT             87
-#define LOAD_FAST__LOAD_FAST             87
+#define LOAD_FAST__LOAD_FAST            131
-#define STORE_FAST__LOAD_FAST           131
+#define STORE_FAST__LOAD_FAST           140
-#define LOAD_FAST__LOAD_CONST           134
+#define LOAD_FAST__LOAD_CONST           141
-#define LOAD_CONST__LOAD_FAST           140
+#define LOAD_CONST__LOAD_FAST           143
-#define STORE_FAST__STORE_FAST          141
+#define STORE_FAST__STORE_FAST          150
 #define DO_TRACING                      255
 #ifdef NEED_OPCODE_JUMP_TABLES
 static uint32_t _PyOpcode_RelativeJump[8] = {
@@ -186,7 +188,7 @@ static uint32_t _PyOpcode_Jump[8] = {
     0U,
     536870912U,
     2316288000U,
-    3U,
+    67U,
     0U,
     0U,
     0U,

Lib/importlib/_bootstrap_external.py

@@ -380,6 +380,7 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.11a4 3472 (bpo-46009: replace GEN_START with POP_TOP)
 #     Python 3.11a4 3473 (Add POP_JUMP_IF_NOT_NONE/POP_JUMP_IF_NONE opcodes)
 #     Python 3.11a4 3474 (Add RESUME opcode)
+#     Python 3.11a5 3475 (Add RETURN_GENERATOR opcode)
 
 #     Python 3.12 will start with magic number 3500
 
@@ -393,7 +394,7 @@ _code_type = type(_write_atomic.__code__)
 # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array
 # in PC/launcher.c must also be updated.
 
-MAGIC_NUMBER = (3474).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3475).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/inspect.py

@@ -1819,11 +1819,11 @@ def getgeneratorstate(generator):
     """
     if generator.gi_running:
         return GEN_RUNNING
+    if generator.gi_suspended:
+        return GEN_SUSPENDED
     if generator.gi_frame is None:
         return GEN_CLOSED
-    if generator.gi_frame.f_lasti == -1:
+    return GEN_CREATED
-        return GEN_CREATED
-    return GEN_SUSPENDED
 
 
 def getgeneratorlocals(generator):
@@ -1861,11 +1861,11 @@ def getcoroutinestate(coroutine):
     """
     if coroutine.cr_running:
         return CORO_RUNNING
+    if coroutine.cr_suspended:
+        return CORO_SUSPENDED
     if coroutine.cr_frame is None:
         return CORO_CLOSED
-    if coroutine.cr_frame.f_lasti == -1:
+    return CORO_CREATED
-        return CORO_CREATED
-    return CORO_SUSPENDED
 
 
 def getcoroutinelocals(coroutine):

Lib/opcode.py

@@ -94,6 +94,7 @@ def_op('LOAD_BUILD_CLASS', 71)
 
 def_op('GET_AWAITABLE', 73)
 def_op('LOAD_ASSERTION_ERROR', 74)
+def_op('RETURN_GENERATOR', 75)
 
 def_op('LIST_TO_TUPLE', 82)
 def_op('RETURN_VALUE', 83)
@@ -155,7 +156,7 @@ def_op('RAISE_VARARGS', 130)    # Number of raise arguments (1, 2, or 3)
 
 def_op('MAKE_FUNCTION', 132)    # Flags
 def_op('BUILD_SLICE', 133)      # Number of items
-
+jabs_op('JUMP_NO_INTERRUPT', 134) # Target byte offset from beginning of code
 def_op('MAKE_CELL', 135)
 hasfree.append(135)
 def_op('LOAD_CLOSURE', 136)

Lib/test/test_compile.py

@@ -954,7 +954,7 @@ if 1:
                     x
                     in
                     y)
-        genexp_lines = [None, 1, 3, 1]
+        genexp_lines = [1, 3, 1]
 
         genexp_code = return_genexp.__code__.co_consts[1]
         code_lines = [ None if line is None else line-return_genexp.__code__.co_firstlineno
@@ -967,7 +967,7 @@ if 1:
             async for i in aseq:
                 body
 
-        expected_lines = [None, 0, 1, 2, 1]
+        expected_lines = [0, 1, 2, 1]
         code_lines = [ None if line is None else line-test.__code__.co_firstlineno
                       for (_, _, line) in test.__code__.co_lines() ]
         self.assertEqual(expected_lines, code_lines)

Lib/test/test_generators.py

@@ -897,7 +897,7 @@ From the Iterators list, about the types of these things.
 >>> type(i)
 <class 'generator'>
 >>> [s for s in dir(i) if not s.startswith('_')]
-['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_yieldfrom', 'send', 'throw']
+['close', 'gi_code', 'gi_frame', 'gi_running', 'gi_suspended', 'gi_yieldfrom', 'send', 'throw']
 >>> from test.support import HAVE_DOCSTRINGS
 >>> print(i.__next__.__doc__ if HAVE_DOCSTRINGS else 'Implement next(self).')
 Implement next(self).

Lib/test/test_sys.py

@@ -1386,7 +1386,7 @@ class SizeofTest(unittest.TestCase):
         def func():
             return sys._getframe()
         x = func()
-        check(x, size('3Pi3c8P2ic?P'))
+        check(x, size('3Pi3c7P2ic??P'))
         # function
         def func(): pass
         check(func, size('14Pi'))
@@ -1403,7 +1403,7 @@ class SizeofTest(unittest.TestCase):
             check(bar, size('PP'))
         # generator
         def get_gen(): yield 1
-        check(get_gen(), size('P2P4P4c8P2ic?P'))
+        check(get_gen(), size('P2P4P4c7P2ic??P'))
         # iterator
         check(iter('abc'), size('lP'))
         # callable-iterator

Misc/NEWS.d/next/Core and Builtins/2022-01-17-12-57-27.bpo-46409.HouS6m.rst

@@ -0,0 +1,6 @@
+Add new ``RETURN_GENERATOR`` bytecode to make generators.
+Simplifies calling Python functions in the VM, as they no
+longer any need to special case generator functions.
+
+Also add ``JUMP_NO_INTERRUPT`` bytecode that acts like
+``JUMP_ABSOLUTE``, but does not check for interrupts.

Objects/frameobject.c

@@ -242,6 +242,7 @@ mark_stacks(PyCodeObject *code_obj, int len)
                     break;
                 }
                 case JUMP_ABSOLUTE:
+                case JUMP_NO_INTERRUPT:
                     j = get_arg(code, i);
                     assert(j < len);
                     if (stacks[j] == UNINITIALIZED && j < i) {
@@ -625,7 +626,7 @@ frame_dealloc(PyFrameObject *f)
 {
     /* It is the responsibility of the owning generator/coroutine
      * to have cleared the generator pointer */
-    assert(f->f_frame->generator == NULL);
+    assert(!f->f_frame->is_generator);
 
     if (_PyObject_GC_IS_TRACKED(f)) {
         _PyObject_GC_UNTRACK(f);
@@ -698,8 +699,11 @@ frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
                         "cannot clear an executing frame");
         return NULL;
     }
-    if (f->f_frame->generator) {
+    if (f->f_frame->is_generator) {
-        _PyGen_Finalize(f->f_frame->generator);
+        assert(!f->f_owns_frame);
+        size_t offset_in_gen = offsetof(PyGenObject, gi_iframe);
+        PyObject *gen = (PyObject *)(((char *)f->f_frame) - offset_in_gen);
+        _PyGen_Finalize(gen);
     }
     (void)frame_tp_clear(f);
     Py_RETURN_NONE;

Objects/genobject.c

@@ -87,7 +87,7 @@ _PyGen_Finalize(PyObject *self)
        issue a RuntimeWarning. */
     if (gen->gi_code != NULL &&
         ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE &&
-        ((InterpreterFrame *)gen->gi_iframe)->f_lasti == -1)
+        ((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_CREATED)
     {
         _PyErr_WarnUnawaitedCoroutine((PyObject *)gen);
     }
@@ -133,7 +133,7 @@ gen_dealloc(PyGenObject *gen)
     if (gen->gi_frame_valid) {
         InterpreterFrame *frame = (InterpreterFrame *)gen->gi_iframe;
         gen->gi_frame_valid = 0;
-        frame->generator = NULL;
+        frame->is_generator = false;
         frame->previous = NULL;
         _PyFrame_Clear(frame);
     }
@@ -156,7 +156,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     PyObject *result;
 
     *presult = NULL;
-    if (frame->f_lasti < 0 && arg && arg != Py_None) {
+    if (frame->f_state == FRAME_CREATED && arg && arg != Py_None) {
         const char *msg = "can't send non-None value to a "
                             "just-started generator";
         if (PyCoro_CheckExact(gen)) {
@@ -265,7 +265,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult,
     /* first clean reference cycle through stored exception traceback */
     _PyErr_ClearExcState(&gen->gi_exc_state);
 
-    frame->generator = NULL;
+    frame->is_generator = false;
     gen->gi_frame_valid = 0;
     _PyFrame_Clear(frame);
     *presult = result;
@@ -753,6 +753,15 @@ gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored))
     return PyBool_FromLong(_PyFrame_IsExecuting((InterpreterFrame *)gen->gi_iframe));
 }
 
+static PyObject *
+gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored))
+{
+    if (gen->gi_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)gen->gi_iframe)->f_state == FRAME_SUSPENDED);
+}
+
 static PyObject *
 _gen_getframe(PyGenObject *gen, const char *const name)
 {
@@ -780,6 +789,7 @@ static PyGetSetDef gen_getsetlist[] = {
      PyDoc_STR("object being iterated by yield from, or None")},
     {"gi_running", (getter)gen_getrunning, NULL, NULL},
     {"gi_frame", (getter)gen_getframe,  NULL, NULL},
+    {"gi_suspended", (getter)gen_getsuspended,  NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -886,22 +896,16 @@ make_gen(PyTypeObject *type, PyFunctionObject *func)
     gen->gi_weakreflist = NULL;
     gen->gi_exc_state.exc_value = NULL;
     gen->gi_exc_state.previous_item = NULL;
-    if (func->func_name != NULL)
+    assert(func->func_name != NULL);
-        gen->gi_name = func->func_name;
+    gen->gi_name = Py_NewRef(func->func_name);
-    else
+    assert(func->func_qualname != NULL);
-        gen->gi_name = gen->gi_code->co_name;
+    gen->gi_qualname = Py_NewRef(func->func_qualname);
-    Py_INCREF(gen->gi_name);
-    if (func->func_qualname != NULL)
-        gen->gi_qualname = func->func_qualname;
-    else
-        gen->gi_qualname = gen->gi_name;
-    Py_INCREF(gen->gi_qualname);
     _PyObject_GC_TRACK(gen);
     return (PyObject *)gen;
 }
 
 static PyObject *
-compute_cr_origin(int origin_depth);
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame);
 
 PyObject *
 _Py_MakeCoro(PyFunctionObject *func)
@@ -935,7 +939,8 @@ _Py_MakeCoro(PyFunctionObject *func)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        assert(_PyEval_GetFrame());
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame()->previous);
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);
@@ -965,7 +970,7 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
     assert(frame->frame_obj == f);
     f->f_owns_frame = 0;
     f->f_frame = frame;
-    frame->generator = (PyObject *) gen;
+    frame->is_generator = true;
     assert(PyObject_GC_IsTracked((PyObject *)f));
     gen->gi_code = PyFrame_GetCode(f);
     Py_INCREF(gen->gi_code);
@@ -1097,6 +1102,15 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored))
     return yf;
 }
 
+static PyObject *
+cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored))
+{
+    if (coro->cr_frame_valid == 0) {
+        Py_RETURN_FALSE;
+    }
+    return PyBool_FromLong(((InterpreterFrame *)coro->cr_iframe)->f_state == FRAME_SUSPENDED);
+}
+
 static PyObject *
 cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored))
 {
@@ -1122,6 +1136,7 @@ static PyGetSetDef coro_getsetlist[] = {
      PyDoc_STR("object being awaited on, or None")},
     {"cr_running", (getter)cr_getrunning, NULL, NULL},
     {"cr_frame", (getter)cr_getframe, NULL, NULL},
+    {"cr_suspended", (getter)cr_getsuspended, NULL, NULL},
     {NULL} /* Sentinel */
 };
 
@@ -1299,9 +1314,9 @@ PyTypeObject _PyCoroWrapper_Type = {
 };
 
 static PyObject *
-compute_cr_origin(int origin_depth)
+compute_cr_origin(int origin_depth, InterpreterFrame *current_frame)
 {
-    InterpreterFrame *frame = _PyEval_GetFrame();
+    InterpreterFrame *frame = current_frame;
     /* First count how many frames we have */
     int frame_count = 0;
     for (; frame && frame_count < origin_depth; ++frame_count) {
@@ -1313,7 +1328,7 @@ compute_cr_origin(int origin_depth)
     if (cr_origin == NULL) {
         return NULL;
     }
-    frame = _PyEval_GetFrame();
+    frame = current_frame;
     for (int i = 0; i < frame_count; ++i) {
         PyCodeObject *code = frame->f_code;
         PyObject *frameinfo = Py_BuildValue("OiO",
@@ -1345,7 +1360,7 @@ PyCoro_New(PyFrameObject *f, PyObject *name, PyObject *qualname)
     if (origin_depth == 0) {
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = NULL;
     } else {
-        PyObject *cr_origin = compute_cr_origin(origin_depth);
+        PyObject *cr_origin = compute_cr_origin(origin_depth, _PyEval_GetFrame());
         ((PyCoroObject *)coro)->cr_origin_or_finalizer = cr_origin;
         if (!cr_origin) {
             Py_DECREF(coro);

Python/ceval.c

@@ -1345,7 +1345,7 @@ eval_frame_handle_pending(PyThreadState *tstate)
 
 #define CHECK_EVAL_BREAKER() \
     if (_Py_atomic_load_relaxed(eval_breaker)) { \
-        goto check_eval_breaker; \
+        goto handle_eval_breaker; \
     }
 
 
@@ -1620,12 +1620,6 @@ trace_function_exit(PyThreadState *tstate, InterpreterFrame *frame, PyObject *re
     return 0;
 }
 
-static PyObject *
-make_coro(PyThreadState *tstate, PyFunctionObject *func,
-          PyObject *locals,
-          PyObject* const* args, size_t argcount,
-          PyObject *kwnames);
-
 static int
 skip_backwards_over_extended_args(PyCodeObject *code, int offset)
 {
@@ -1760,49 +1754,21 @@ resume_frame:
     assert(!_PyErr_Occurred(tstate));
 #endif
 
-check_eval_breaker:
-    {
-        assert(STACK_LEVEL() >= 0); /* else underflow */
-        assert(STACK_LEVEL() <= frame->f_code->co_stacksize);  /* else overflow */
-        assert(!_PyErr_Occurred(tstate));
-
-        /* Do periodic things.  Doing this every time through
-           the loop would add too much overhead, so we do it
-           only every Nth instruction.  We also do it if
-           ``pending.calls_to_do'' is set, i.e. when an asynchronous
-           event needs attention (e.g. a signal handler or
-           async I/O handler); see Py_AddPendingCall() and
-           Py_MakePendingCalls() above. */
-
-        if (_Py_atomic_load_relaxed(eval_breaker)) {
-            opcode = _Py_OPCODE(*next_instr);
-            if (opcode != BEFORE_ASYNC_WITH &&
-                opcode != SEND &&
-                _Py_OPCODE(next_instr[-1]) != SEND) {
-                /* Few cases where we skip running signal handlers and other
-                   pending calls:
-                   - If we're about to enter the 'with:'. It will prevent
-                     emitting a resource warning in the common idiom
-                     'with open(path) as file:'.
-                   - If we're about to enter the 'async with:'.
-                   - If we're about to enter the 'try:' of a try/finally (not
-                     *very* useful, but might help in some cases and it's
-                     traditional)
-                   - If we're resuming a chain of nested 'yield from' or
-                     'await' calls, then each frame is parked with YIELD_FROM
-                     as its next opcode. If the user hit control-C we want to
-                     wait until we've reached the innermost frame before
-                     running the signal handler and raising KeyboardInterrupt
-                     (see bpo-30039).
-                */
-                if (eval_frame_handle_pending(tstate) != 0) {
-                    goto error;
-                }
-             }
-        }
-
     DISPATCH();
 
+handle_eval_breaker:
+
+    /* Do periodic things, like check for signals and async I/0.
+     * We need to do reasonably frequently, but not too frequently.
+     * All loops should include a check of the eval breaker.
+     * We also check on return from any builtin function.
+     */
+    if (eval_frame_handle_pending(tstate) != 0) {
+        goto error;
+    }
+    DISPATCH();
+
+    {
     /* Start instructions */
 #if USE_COMPUTED_GOTOS
     {
@@ -1834,6 +1800,9 @@ check_eval_breaker:
                 next_instr = first_instr + nexti;
             }
             frame->f_state = FRAME_EXECUTING;
+            if (_Py_atomic_load_relaxed(eval_breaker) && oparg < 2) {
+                goto handle_eval_breaker;
+            }
             DISPATCH();
         }
 
@@ -4152,6 +4121,17 @@ check_eval_breaker:
             DISPATCH();
         }
 
+        TARGET(JUMP_NO_INTERRUPT) {
+            /* This bytecode is used in the `yield from` or `await` loop.
+             * If there is an interrupt, we want it handled in the innermost
+             * generator or coroutine, so we deliberately do not check it here.
+             * (see bpo-30039).
+             */
+            frame->f_state = FRAME_EXECUTING;
+            JUMPTO(oparg);
+            DISPATCH();
+        }
+
         TARGET(JUMP_ABSOLUTE_QUICK) {
             assert(oparg < INSTR_OFFSET());
             JUMPTO(oparg);
@@ -4627,28 +4607,25 @@ check_eval_breaker:
             // Check if the call can be inlined or not
             if (Py_TYPE(function) == &PyFunction_Type && tstate->interp->eval_frame == NULL) {
                 int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(function))->co_flags;
-                int is_generator = code_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
+                PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
-                if (!is_generator) {
+                STACK_SHRINK(oparg);
-                    PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : PyFunction_GET_GLOBALS(function);
+                InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
-                    STACK_SHRINK(oparg);
+                    tstate, (PyFunctionObject *)function, locals,
-                    InterpreterFrame *new_frame = _PyEvalFramePushAndInit(
+                    stack_pointer, nargs, kwnames
-                        tstate, (PyFunctionObject *)function, locals,
+                );
-                        stack_pointer, nargs, kwnames
+                STACK_SHRINK(postcall_shrink);
-                    );
+                RESET_STACK_ADJUST_FOR_CALLS;
-                    STACK_SHRINK(postcall_shrink);
+                // The frame has stolen all the arguments from the stack,
-                    RESET_STACK_ADJUST_FOR_CALLS;
+                // so there is no need to clean them up.
-                    // The frame has stolen all the arguments from the stack,
+                Py_XDECREF(kwnames);
-                    // so there is no need to clean them up.
+                Py_DECREF(function);
-                    Py_XDECREF(kwnames);
+                if (new_frame == NULL) {
-                    Py_DECREF(function);
+                    goto error;
-                    if (new_frame == NULL) {
-                        goto error;
-                    }
-                    _PyFrame_SetStackPointer(frame, stack_pointer);
-                    new_frame->previous = frame;
-                    cframe.current_frame = frame = new_frame;
-                    goto start_frame;
                 }
+                _PyFrame_SetStackPointer(frame, stack_pointer);
+                new_frame->previous = frame;
+                cframe.current_frame = frame = new_frame;
+                goto start_frame;
             }
             /* Callable is not a normal Python function */
             PyObject *res;
@@ -5076,6 +5053,40 @@ check_eval_breaker:
             DISPATCH();
         }
 
+        TARGET(RETURN_GENERATOR) {
+            PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(frame->f_func);
+            if (gen == NULL) {
+                goto error;
+            }
+            assert(EMPTY());
+            _PyFrame_SetStackPointer(frame, stack_pointer);
+            InterpreterFrame *gen_frame = (InterpreterFrame *)gen->gi_iframe;
+            _PyFrame_Copy(frame, gen_frame);
+            assert(frame->frame_obj == NULL);
+            gen->gi_frame_valid = 1;
+            gen_frame->is_generator = true;
+            gen_frame->f_state = FRAME_CREATED;
+            _Py_LeaveRecursiveCall(tstate);
+            if (!frame->is_entry) {
+                InterpreterFrame *prev = frame->previous;
+                _PyThreadState_PopFrame(tstate, frame);
+                frame = cframe.current_frame = prev;
+                _PyFrame_StackPush(frame, (PyObject *)gen);
+                goto resume_frame;
+            }
+            /* Make sure that frame is in a valid state */
+            frame->stacktop = 0;
+            frame->f_locals = NULL;
+            Py_INCREF(frame->f_func);
+            Py_INCREF(frame->f_code);
+            /* Restore previous cframe and return. */
+            tstate->cframe = cframe.previous;
+            tstate->cframe->use_tracing = cframe.use_tracing;
+            assert(tstate->cframe->current_frame == frame->previous);
+            assert(!_PyErr_Occurred(tstate));
+            return (PyObject *)gen;
+        }
+
         TARGET(BUILD_SLICE) {
             PyObject *start, *stop, *step, *slice;
             if (oparg == 3)
@@ -5222,11 +5233,14 @@ check_eval_breaker:
             frame->f_lasti = INSTR_OFFSET();
             TRACING_NEXTOPARG();
             if (opcode == RESUME) {
+                if (oparg < 2) {
+                    CHECK_EVAL_BREAKER();
+                }
                 /* Call tracing */
                 TRACE_FUNCTION_ENTRY();
                 DTRACE_FUNCTION_ENTRY();
             }
-            else {
+            else if (frame->f_state > FRAME_CREATED) {
                 /* line-by-line tracing support */
                 if (PyDTrace_LINE_ENABLED()) {
                     maybe_dtrace_line(frame, &tstate->trace_info, instr_prev);
@@ -5961,33 +5975,6 @@ fail_post_args:
     return -1;
 }
 
-/* Consumes all the references to the args */
-static PyObject *
-make_coro(PyThreadState *tstate, PyFunctionObject *func,
-          PyObject *locals,
-          PyObject* const* args, size_t argcount,
-          PyObject *kwnames)
-{
-    assert (((PyCodeObject *)func->func_code)->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR));
-    PyObject *gen = _Py_MakeCoro(func);
-    if (gen == NULL) {
-        return NULL;
-    }
-    InterpreterFrame *frame = (InterpreterFrame *)((PyGenObject *)gen)->gi_iframe;
-    PyCodeObject *code = (PyCodeObject *)func->func_code;
-    _PyFrame_InitializeSpecials(frame, func, locals, code->co_nlocalsplus);
-    for (int i = 0; i < code->co_nlocalsplus; i++) {
-        frame->localsplus[i] = NULL;
-    }
-    ((PyGenObject *)gen)->gi_frame_valid = 1;
-    if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) {
-        Py_DECREF(gen);
-        return NULL;
-    }
-    frame->generator = gen;
-    return gen;
-}
-
 /* Consumes all the references to the args */
 static InterpreterFrame *
 _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func,
@@ -6041,10 +6028,7 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
                PyObject* const* args, size_t argcount,
                PyObject *kwnames)
 {
-    PyCodeObject *code = (PyCodeObject *)func->func_code;
+    /* _PyEvalFramePushAndInit consumes all the references to its arguments */
-    /* _PyEvalFramePushAndInit and make_coro consume
-     * all the references to their arguments
-     */
     for (size_t i = 0; i < argcount; i++) {
         Py_INCREF(args[i]);
     }
@@ -6054,19 +6038,16 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func,
             Py_INCREF(args[i+argcount]);
         }
     }
-    int is_coro = code->co_flags &
-        (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR);
-    if (is_coro) {
-        return make_coro(tstate, func, locals, args, argcount, kwnames);
-    }
     InterpreterFrame *frame = _PyEvalFramePushAndInit(
         tstate, func, locals, args, argcount, kwnames);
     if (frame == NULL) {
         return NULL;
     }
     PyObject *retval = _PyEval_EvalFrame(tstate, frame, 0);
-    assert(frame->stacktop >= 0);
+    assert(
-    assert(_PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame));
+        _PyFrame_GetStackPointer(frame) == _PyFrame_Stackbase(frame) ||
+        _PyFrame_GetStackPointer(frame) == frame->localsplus
+    );
     _PyEvalFrameClearAndPop(tstate, frame);
     return retval;
 }

Python/compile.c

@@ -969,6 +969,7 @@ stack_effect(int opcode, int oparg, int jump)
         /* Jumps */
         case JUMP_FORWARD:
         case JUMP_ABSOLUTE:
+        case JUMP_NO_INTERRUPT:
             return 0;
 
         case JUMP_IF_TRUE_OR_POP:
@@ -1017,6 +1018,9 @@ stack_effect(int opcode, int oparg, int jump)
         case DELETE_FAST:
             return 0;
 
+        case RETURN_GENERATOR:
+            return 0;
+
         case RAISE_VARARGS:
             return -oparg;
 
@@ -1841,7 +1845,7 @@ compiler_add_yield_from(struct compiler *c, int await)
     ADDOP_JUMP(c, SEND, exit);
     compiler_use_next_block(c, resume);
     ADDOP_I(c, RESUME, await ? 3 : 2);
-    ADDOP_JUMP(c, JUMP_ABSOLUTE, start);
+    ADDOP_JUMP(c, JUMP_NO_INTERRUPT, start);
     compiler_use_next_block(c, exit);
     return 1;
 }
@@ -7055,6 +7059,7 @@ stackdepth(struct compiler *c)
             }
             depth = new_depth;
             if (instr->i_opcode == JUMP_ABSOLUTE ||
+                instr->i_opcode == JUMP_NO_INTERRUPT ||
                 instr->i_opcode == JUMP_FORWARD ||
                 instr->i_opcode == RETURN_VALUE ||
                 instr->i_opcode == RAISE_VARARGS ||
@@ -7572,9 +7577,6 @@ normalize_jumps(struct assembler *a)
             if (last->i_target->b_visited == 0) {
                 last->i_opcode = JUMP_FORWARD;
             }
-            else if (b->b_iused >= 2 && b->b_instr[b->b_iused-2].i_opcode == SEND) {
-                last->i_opcode = JUMP_ABSOLUTE_QUICK;
-            }
         }
     }
 }
@@ -7998,6 +8000,34 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
     }
     assert(c->u->u_firstlineno > 0);
 
+    /* Add the generator prefix instructions. */
+    if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
+        struct instr make_gen = {
+            .i_opcode = RETURN_GENERATOR,
+            .i_oparg = 0,
+            .i_lineno = c->u->u_firstlineno,
+            .i_col_offset = -1,
+            .i_end_lineno = c->u->u_firstlineno,
+            .i_end_col_offset = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, 0, &make_gen) < 0) {
+            return -1;
+        }
+        struct instr pop_top = {
+            .i_opcode = POP_TOP,
+            .i_oparg = 0,
+            .i_lineno = -1,
+            .i_col_offset = -1,
+            .i_end_lineno = -1,
+            .i_end_col_offset = -1,
+            .i_target = NULL,
+        };
+        if (insert_instruction(entryblock, 1, &pop_top) < 0) {
+            return -1;
+        }
+    }
+
     /* Set up cells for any variable that escapes, to be put in a closure. */
     const int ncellvars = (int)PyDict_GET_SIZE(c->u->u_cellvars);
     if (ncellvars) {
@@ -8036,22 +8066,6 @@ insert_prefix_instructions(struct compiler *c, basicblock *entryblock,
         PyMem_RawFree(sorted);
     }
 
-    /* Add the generator prefix instructions. */
-    if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
-        struct instr pop_top = {
-            .i_opcode = POP_TOP,
-            .i_oparg = 0,
-            .i_lineno = -1,
-            .i_col_offset = -1,
-            .i_end_lineno = -1,
-            .i_end_col_offset = -1,
-            .i_target = NULL,
-        };
-        if (insert_instruction(entryblock, 0, &pop_top) < 0) {
-            return -1;
-        }
-    }
-
     if (nfreevars) {
         struct instr copy_frees = {
             .i_opcode = COPY_FREE_VARS,
@@ -8801,6 +8815,7 @@ normalize_basic_block(basicblock *bb) {
                 break;
             case JUMP_ABSOLUTE:
             case JUMP_FORWARD:
+            case JUMP_NO_INTERRUPT:
                 bb->b_nofallthrough = 1;
                 /* fall through */
             case POP_JUMP_IF_NOT_NONE:
@@ -8985,6 +9000,7 @@ optimize_cfg(struct compiler *c, struct assembler *a, PyObject *consts)
         if (b->b_iused > 0) {
             struct instr *b_last_instr = &b->b_instr[b->b_iused - 1];
             if (b_last_instr->i_opcode == JUMP_ABSOLUTE ||
+                b_last_instr->i_opcode == JUMP_NO_INTERRUPT ||
                 b_last_instr->i_opcode == JUMP_FORWARD) {
                 if (b_last_instr->i_target == b->b_next) {
                     assert(b->b_next->b_iused);

Python/frame.c

@@ -3,6 +3,7 @@
 #include "frameobject.h"
 #include "pycore_frame.h"
 #include "pycore_object.h"        // _PyObject_GC_UNTRACK()
+#include "opcode.h"
 
 int
 _PyFrame_Traverse(InterpreterFrame *frame, visitproc visit, void *arg)
@@ -51,15 +52,6 @@ _PyFrame_Copy(InterpreterFrame *src, InterpreterFrame *dest)
     memcpy(dest, src, size);
 }
 
-static inline void
-clear_specials(InterpreterFrame *frame)
-{
-    frame->generator = NULL;
-    Py_XDECREF(frame->frame_obj);
-    Py_XDECREF(frame->f_locals);
-    Py_DECREF(frame->f_func);
-    Py_DECREF(frame->f_code);
-}
 
 static void
 take_ownership(PyFrameObject *f, InterpreterFrame *frame)
@@ -94,8 +86,8 @@ void
 _PyFrame_Clear(InterpreterFrame * frame)
 {
     /* It is the responsibility of the owning generator/coroutine
-     * to have cleared the generator pointer */
+     * to have cleared the enclosing generator, if any. */
-    assert(frame->generator == NULL);
+    assert(!frame->is_generator);
     if (frame->frame_obj) {
         PyFrameObject *f = frame->frame_obj;
         frame->frame_obj = NULL;
@@ -110,5 +102,8 @@ _PyFrame_Clear(InterpreterFrame * frame)
     for (int i = 0; i < frame->stacktop; i++) {
         Py_XDECREF(frame->localsplus[i]);
     }
-    clear_specials(frame);
+    Py_XDECREF(frame->frame_obj);
+    Py_XDECREF(frame->f_locals);
+    Py_DECREF(frame->f_func);
+    Py_DECREF(frame->f_code);
 }

Python/opcode_targets.h

@@ -74,19 +74,19 @@ static void *opcode_targets[256] = {
     &&TARGET_LOAD_METHOD_CACHED,
     &&TARGET_GET_AWAITABLE,
     &&TARGET_LOAD_ASSERTION_ERROR,
+    &&TARGET_RETURN_GENERATOR,
     &&TARGET_LOAD_METHOD_CLASS,
     &&TARGET_LOAD_METHOD_MODULE,
     &&TARGET_LOAD_METHOD_NO_DICT,
     &&TARGET_STORE_ATTR_ADAPTIVE,
     &&TARGET_STORE_ATTR_INSTANCE_VALUE,
     &&TARGET_STORE_ATTR_SLOT,
-    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_LIST_TO_TUPLE,
     &&TARGET_RETURN_VALUE,
     &&TARGET_IMPORT_STAR,
     &&TARGET_SETUP_ANNOTATIONS,
     &&TARGET_YIELD_VALUE,
-    &&TARGET_LOAD_FAST__LOAD_FAST,
+    &&TARGET_STORE_ATTR_WITH_HINT,
     &&TARGET_PREP_RERAISE_STAR,
     &&TARGET_POP_EXCEPT,
     &&TARGET_STORE_NAME,
@@ -130,26 +130,26 @@ static void *opcode_targets[256] = {
     &&TARGET_POP_JUMP_IF_NOT_NONE,
     &&TARGET_POP_JUMP_IF_NONE,
     &&TARGET_RAISE_VARARGS,
-    &&TARGET_STORE_FAST__LOAD_FAST,
+    &&TARGET_LOAD_FAST__LOAD_FAST,
     &&TARGET_MAKE_FUNCTION,
     &&TARGET_BUILD_SLICE,
-    &&TARGET_LOAD_FAST__LOAD_CONST,
+    &&TARGET_JUMP_NO_INTERRUPT,
     &&TARGET_MAKE_CELL,
     &&TARGET_LOAD_CLOSURE,
     &&TARGET_LOAD_DEREF,
     &&TARGET_STORE_DEREF,
     &&TARGET_DELETE_DEREF,
-    &&TARGET_LOAD_CONST__LOAD_FAST,
+    &&TARGET_STORE_FAST__LOAD_FAST,
-    &&TARGET_STORE_FAST__STORE_FAST,
+    &&TARGET_LOAD_FAST__LOAD_CONST,
     &&TARGET_CALL_FUNCTION_EX,
-    &&_unknown_opcode,
+    &&TARGET_LOAD_CONST__LOAD_FAST,
     &&TARGET_EXTENDED_ARG,
     &&TARGET_LIST_APPEND,
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
     &&TARGET_COPY_FREE_VARS,
-    &&_unknown_opcode,
+    &&TARGET_STORE_FAST__STORE_FAST,
     &&TARGET_RESUME,
     &&TARGET_MATCH_CLASS,
     &&_unknown_opcode,

Python/specialize.c

@@ -499,7 +499,6 @@ initial_counter_value(void) {
 #define SPEC_FAIL_DIFFERENT_TYPES 12
 
 /* Calls */
-#define SPEC_FAIL_GENERATOR 7
 #define SPEC_FAIL_COMPLEX_PARAMETERS 8
 #define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 9
 #define SPEC_FAIL_CO_NOT_OPTIMIZED 10
@@ -1153,9 +1152,6 @@ _Py_IDENTIFIER(__getitem__);
 static int
 function_kind(PyCodeObject *code) {
     int flags = code->co_flags;
-    if (flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
-        return SPEC_FAIL_GENERATOR;
-    }
     if ((flags & (CO_VARKEYWORDS | CO_VARARGS)) || code->co_kwonlyargcount) {
         return SPEC_FAIL_COMPLEX_PARAMETERS;
     }