[3.11] GH-97779: Ensure that *all* frame objects are backed by "complete" frames (GH-97886)

(cherry picked from commit 0ff8fd6583)

Co-authored-by: Brandt Bucher <brandtbucher@microsoft.com>

Lib/test/test_code.py

@@ -132,6 +132,7 @@ import doctest
 import unittest
 import textwrap
 import weakref
+import dis
 
 try:
     import ctypes
@@ -671,6 +672,38 @@ class CodeLocationTest(unittest.TestCase):
         self.check_lines(misshappen)
         self.check_lines(bug93662)
 
+    @cpython_only
+    def test_code_new_empty(self):
+        # If this test fails, it means that the construction of PyCode_NewEmpty
+        # needs to be modified! Please update this test *and* PyCode_NewEmpty,
+        # so that they both stay in sync.
+        def f():
+            pass
+        PY_CODE_LOCATION_INFO_NO_COLUMNS = 13
+        f.__code__ = f.__code__.replace(
+            co_firstlineno=42,
+            co_code=bytes(
+                [
+                    dis.opmap["RESUME"], 0,
+                    dis.opmap["LOAD_ASSERTION_ERROR"], 0,
+                    dis.opmap["RAISE_VARARGS"], 1,
+                ]
+            ),
+            co_linetable=bytes(
+                [
+                    (1 << 7)
+                    | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3)
+                    | (3 - 1),
+                    0,
+                ]
+            ),
+        )
+        self.assertRaises(AssertionError, f)
+        self.assertEqual(
+            list(f.__code__.co_positions()),
+            3 * [(42, 42, None, None)],
+        )
+
 
 if check_impl_detail(cpython=True) and ctypes is not None:
     py = ctypes.pythonapi

Misc/NEWS.d/next/Core and Builtins/2022-10-04-02-00-10.gh-issue-97779.f3N1hI.rst

@@ -0,0 +1 @@
+Ensure that all Python frame objects are backed by "complete" frames.

Objects/codeobject.c

@@ -636,12 +636,22 @@ PyCode_New(int argcount, int kwonlyargcount,
                                      exceptiontable);
 }
 
-static const char assert0[6] = {
+// NOTE: When modifying the construction of PyCode_NewEmpty, please also change
+// test.test_code.CodeLocationTest.test_code_new_empty to keep it in sync!
+
+static const uint8_t assert0[6] = {
     RESUME, 0,
     LOAD_ASSERTION_ERROR, 0,
     RAISE_VARARGS, 1
 };
 
+static const uint8_t linetable[2] = {
+    (1 << 7)  // New entry.
+    | (PY_CODE_LOCATION_INFO_NO_COLUMNS << 3)
+    | (3 - 1),  // Three code units.
+    0,  // Offset from co_firstlineno.
+};
+
 PyCodeObject *
 PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 {
@@ -649,6 +659,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
     PyObject *filename_ob = NULL;
     PyObject *funcname_ob = NULL;
     PyObject *code_ob = NULL;
+    PyObject *linetable_ob = NULL;
     PyCodeObject *result = NULL;
 
     nulltuple = PyTuple_New(0);
@@ -663,10 +674,14 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
     if (filename_ob == NULL) {
         goto failed;
     }
-    code_ob = PyBytes_FromStringAndSize(assert0, 6);
+    code_ob = PyBytes_FromStringAndSize((const char *)assert0, 6);
     if (code_ob == NULL) {
         goto failed;
     }
+    linetable_ob = PyBytes_FromStringAndSize((const char *)linetable, 2);
+    if (linetable_ob == NULL) {
+        goto failed;
+    }
 
 #define emptystring (PyObject *)&_Py_SINGLETON(bytes_empty)
     struct _PyCodeConstructor con = {
@@ -675,7 +690,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
         .qualname = funcname_ob,
         .code = code_ob,
         .firstlineno = firstlineno,
-        .linetable = emptystring,
+        .linetable = linetable_ob,
         .consts = nulltuple,
         .names = nulltuple,
         .localsplusnames = nulltuple,
@@ -690,6 +705,7 @@ failed:
     Py_XDECREF(funcname_ob);
     Py_XDECREF(filename_ob);
     Py_XDECREF(code_ob);
+    Py_XDECREF(linetable_ob);
     return result;
 }
 

Objects/frameobject.c

@@ -590,6 +590,7 @@ first_line_not_before(int *lines, int len, int line)
 static PyFrameState
 _PyFrame_GetState(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     if (frame->f_frame->stacktop == 0) {
         return FRAME_CLEARED;
     }
@@ -1063,6 +1064,9 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
     init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals);
     f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data;
     f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT;
+    // This frame needs to be "complete", so pretend that the first RESUME ran:
+    f->f_frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable;
+    assert(!_PyFrame_IsIncomplete(f->f_frame));
     Py_DECREF(func);
     _PyObject_GC_TRACK(f);
     return f;
@@ -1189,6 +1193,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
 int
 PyFrame_FastToLocalsWithError(PyFrameObject *f)
 {
+    assert(!_PyFrame_IsIncomplete(f->f_frame));
     if (f == NULL) {
         PyErr_BadInternalCall();
         return -1;
@@ -1204,7 +1209,7 @@ void
 PyFrame_FastToLocals(PyFrameObject *f)
 {
     int res;
-
+    assert(!_PyFrame_IsIncomplete(f->f_frame));
     assert(!PyErr_Occurred());
 
     res = PyFrame_FastToLocalsWithError(f);
@@ -1282,6 +1287,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear)
 void
 PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 {
+    assert(!_PyFrame_IsIncomplete(f->f_frame));
     if (f && f->f_fast_as_locals && _PyFrame_GetState(f) != FRAME_CLEARED) {
         _PyFrame_LocalsToFast(f->f_frame, clear);
         f->f_fast_as_locals = 0;
@@ -1292,6 +1298,7 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 int _PyFrame_IsEntryFrame(PyFrameObject *frame)
 {
     assert(frame != NULL);
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     return frame->f_frame->is_entry;
 }
 
@@ -1300,6 +1307,7 @@ PyCodeObject *
 PyFrame_GetCode(PyFrameObject *frame)
 {
     assert(frame != NULL);
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     PyCodeObject *code = frame->f_frame->f_code;
     assert(code != NULL);
     Py_INCREF(code);
@@ -1311,6 +1319,7 @@ PyFrameObject*
 PyFrame_GetBack(PyFrameObject *frame)
 {
     assert(frame != NULL);
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     PyFrameObject *back = frame->f_back;
     if (back == NULL) {
         _PyInterpreterFrame *prev = frame->f_frame->previous;
@@ -1328,24 +1337,28 @@ PyFrame_GetBack(PyFrameObject *frame)
 PyObject*
 PyFrame_GetLocals(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     return frame_getlocals(frame, NULL);
 }
 
 PyObject*
 PyFrame_GetGlobals(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     return frame_getglobals(frame, NULL);
 }
 
 PyObject*
 PyFrame_GetBuiltins(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     return frame_getbuiltins(frame, NULL);
 }
 
 int
 PyFrame_GetLasti(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     int lasti = _PyInterpreterFrame_LASTI(frame->f_frame);
     if (lasti < 0) {
         return -1;
@@ -1356,6 +1369,7 @@ PyFrame_GetLasti(PyFrameObject *frame)
 PyObject *
 PyFrame_GetGenerator(PyFrameObject *frame)
 {
+    assert(!_PyFrame_IsIncomplete(frame->f_frame));
     if (frame->f_frame->owner != FRAME_OWNED_BY_GENERATOR) {
         return NULL;
     }

Python/frame.c

@@ -68,6 +68,13 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame)
     frame = (_PyInterpreterFrame *)f->_f_frame_data;
     f->f_frame = frame;
     frame->owner = FRAME_OWNED_BY_FRAME_OBJECT;
+    if (_PyFrame_IsIncomplete(frame)) {
+        // This may be a newly-created generator or coroutine frame. Since it's
+        // dead anyways, just pretend that the first RESUME ran:
+        PyCodeObject *code = frame->f_code;
+        frame->prev_instr = _PyCode_CODE(code) + code->_co_firsttraceable;
+    }
+    assert(!_PyFrame_IsIncomplete(frame));
     assert(f->f_back == NULL);
     _PyInterpreterFrame *prev = frame->previous;
     while (prev && _PyFrame_IsIncomplete(prev)) {