gh-106597: Add debugging struct with offsets for out-of-process tools (#106598)

Include/internal/pycore_runtime.h

@@ -53,12 +53,101 @@ typedef struct _Py_AuditHookEntry {
     void *userData;
 } _Py_AuditHookEntry;
 
+typedef struct _Py_DebugOffsets {
+    char cookie[8];
+    uint64_t version;
+    // Runtime state offset;
+    struct _runtime_state {
+        off_t finalizing;
+        off_t interpreters_head;
+    } runtime_state;
+
+    // Interpreter state offset;
+    struct _interpreter_state {
+        off_t next;
+        off_t threads_head;
+        off_t gc;
+        off_t imports_modules;
+        off_t sysdict;
+        off_t builtins;
+        off_t ceval_gil;
+        off_t gil_runtime_state_locked;
+        off_t gil_runtime_state_holder;
+    } interpreter_state;
+
+    // Thread state offset;
+    struct _thread_state{
+        off_t prev;
+        off_t next;
+        off_t interp;
+        off_t cframe;
+        off_t thread_id;
+        off_t native_thread_id;
+    } thread_state;
+
+    // InterpreterFrame offset;
+    struct _interpreter_frame {
+        off_t previous;
+        off_t executable;
+        off_t prev_instr;
+        off_t localsplus;
+        off_t owner;
+    } interpreter_frame;
+
+    // CFrame offset;
+    struct _cframe {
+        off_t current_frame;
+        off_t previous;
+    } cframe;
+
+    // Code object offset;
+    struct _code_object {
+        off_t filename;
+        off_t name;
+        off_t linetable;
+        off_t firstlineno;
+        off_t argcount;
+        off_t localsplusnames;
+        off_t localspluskinds;
+        off_t co_code_adaptive;
+    } code_object;
+
+    // PyObject offset;
+    struct _pyobject {
+        off_t ob_type;
+    } pyobject;
+
+    // PyTypeObject object offset;
+    struct _type_object {
+        off_t tp_name;
+    } type_object;
+
+    // PyTuple object offset;
+    struct _tuple_object {
+        off_t ob_item;
+    } tuple_object;
+} _Py_DebugOffsets;
+
 /* Full Python runtime state */
 
 /* _PyRuntimeState holds the global state for the CPython runtime.
    That data is exposed in the internal API as a static variable (_PyRuntime).
    */
 typedef struct pyruntimestate {
+    /* This field must be first to facilitate locating it by out of process
+     * debuggers. Out of process debuggers will use the offsets contained in this
+     * field to be able to locate other fields in several interpreter structures
+     * in a way that doesn't require them to know the exact layout of those
+     * structures.
+     *
+     * IMPORTANT:
+     * This struct is **NOT** backwards compatible between minor version of the
+     * interpreter and the members, order of members and size can change between
+     * minor versions. This struct is only guaranteed to be stable between patch
+     * versions for a given minor version of the interpreter.
+     */
+    _Py_DebugOffsets debug_offsets;
+
     /* Has been initialized to a safe state.
 
        In order to be effective, this must be set to 0 during or right

Include/internal/pycore_runtime_init.h

@@ -21,9 +21,65 @@ extern PyTypeObject _PyExc_MemoryError;
 /* The static initializers defined here should only be used
    in the runtime init code (in pystate.c and pylifecycle.c). */
 
-
 #define _PyRuntimeState_INIT(runtime) \
     { \
+        .debug_offsets = { \
+            .cookie = "xdebugpy", \
+            .version = PY_VERSION_HEX, \
+            .runtime_state = { \
+                .finalizing = offsetof(_PyRuntimeState, _finalizing), \
+                .interpreters_head = offsetof(_PyRuntimeState, interpreters.head), \
+            }, \
+            .interpreter_state = { \
+                .next = offsetof(PyInterpreterState, next), \
+                .threads_head = offsetof(PyInterpreterState, threads.head), \
+                .gc = offsetof(PyInterpreterState, gc), \
+                .imports_modules = offsetof(PyInterpreterState, imports.modules), \
+                .sysdict = offsetof(PyInterpreterState, sysdict), \
+                .builtins = offsetof(PyInterpreterState, builtins), \
+                .ceval_gil = offsetof(PyInterpreterState, ceval.gil), \
+                .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \
+                .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \
+            }, \
+            .thread_state = { \
+                .prev = offsetof(PyThreadState, prev), \
+                .next = offsetof(PyThreadState, next), \
+                .interp = offsetof(PyThreadState, interp), \
+                .cframe = offsetof(PyThreadState, cframe), \
+                .thread_id = offsetof(PyThreadState, thread_id), \
+                .native_thread_id = offsetof(PyThreadState, native_thread_id), \
+            }, \
+            .interpreter_frame = { \
+                .previous = offsetof(_PyInterpreterFrame, previous), \
+                .executable = offsetof(_PyInterpreterFrame, f_executable), \
+                .prev_instr = offsetof(_PyInterpreterFrame, prev_instr), \
+                .localsplus = offsetof(_PyInterpreterFrame, localsplus), \
+                .owner = offsetof(_PyInterpreterFrame, owner), \
+            }, \
+            .cframe = { \
+                .current_frame = offsetof(_PyCFrame, current_frame), \
+                .previous = offsetof(_PyCFrame, previous), \
+            }, \
+            .code_object = { \
+                .filename = offsetof(PyCodeObject, co_filename), \
+                .name = offsetof(PyCodeObject, co_name), \
+                .linetable = offsetof(PyCodeObject, co_linetable), \
+                .firstlineno = offsetof(PyCodeObject, co_firstlineno), \
+                .argcount = offsetof(PyCodeObject, co_argcount), \
+                .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \
+                .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \
+                .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \
+            }, \
+            .pyobject = { \
+                .ob_type = offsetof(PyObject, ob_type), \
+            }, \
+            .type_object = { \
+                .tp_name = offsetof(PyTypeObject, tp_name), \
+            }, \
+            .tuple_object = { \
+                .ob_item = offsetof(PyTupleObject, ob_item), \
+            }, \
+        }, \
         .allocators = { \
             .standard = _pymem_allocators_standard_INIT(runtime), \
             .debug = _pymem_allocators_debug_INIT, \

Misc/NEWS.d/next/Core and Builtins/2023-07-10-15-30-45.gh-issue-106597.WAZ14y.rst

@@ -0,0 +1,5 @@
+A new debug structure of offsets has been added to the ``_PyRuntimeState``
+that will help out-of-process debuggers and profilers to obtain the offsets
+to relevant interpreter structures in a way that is agnostic of how Python
+was compiled and that doesn't require copying the headers. Patch by Pablo
+Galindo