[3.12] gh-76785: Use Pending Calls When Releasing Cross-Interpreter Data (gh-109556) (gh-112288)

This fixes some crashes in the _xxinterpchannels module, due to a race between interpreters.
(cherry picked from commit fd7e08a6f3)

Include/internal/pycore_pystate.h

@@ -148,6 +148,8 @@ extern PyStatus _PyInterpreterState_DeleteExceptMain(_PyRuntimeState *runtime);
 extern void _PySignal_AfterFork(void);
 #endif
 
+PyAPI_FUNC(int) _PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *);
+
 
 PyAPI_FUNC(int) _PyState_AddModule(
     PyThreadState *tstate,

Modules/_xxinterpchannelsmodule.c

@@ -2,8 +2,13 @@
 /* interpreters module */
 /* low-level access to interpreter primitives */
 
+#ifndef Py_BUILD_CORE_BUILTIN
+#  define Py_BUILD_CORE_MODULE 1
+#endif
+
 #include "Python.h"
 #include "interpreteridobject.h"
+#include "pycore_pystate.h"       // _PyCrossInterpreterData_ReleaseAndRawFree()
 
 
 /*
@@ -161,14 +166,24 @@ add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared)
     return cls;
 }
 
+#define XID_IGNORE_EXC 1
+#define XID_FREE 2
+
 static int
-_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc)
+_release_xid_data(_PyCrossInterpreterData *data, int flags)
 {
+    int ignoreexc = flags & XID_IGNORE_EXC;
     PyObject *exc;
     if (ignoreexc) {
         exc = PyErr_GetRaisedException();
     }
-    int res = _PyCrossInterpreterData_Release(data);
+    int res;
+    if (flags & XID_FREE) {
+        res = _PyCrossInterpreterData_ReleaseAndRawFree(data);
+    }
+    else {
+        res = _PyCrossInterpreterData_Release(data);
+    }
     if (res < 0) {
         /* The owning interpreter is already destroyed. */
         if (ignoreexc) {
@@ -176,6 +191,9 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc)
             PyErr_Clear();
         }
     }
+    if (flags & XID_FREE) {
+        /* Either way, we free the data. */
+    }
     if (ignoreexc) {
         PyErr_SetRaisedException(exc);
     }
@@ -367,9 +385,8 @@ static void
 _channelitem_clear(_channelitem *item)
 {
     if (item->data != NULL) {
-        (void)_release_xid_data(item->data, 1);
         // It was allocated in _channel_send().
-        GLOBAL_FREE(item->data);
+        (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE);
         item->data = NULL;
     }
     item->next = NULL;
@@ -1440,14 +1457,12 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res)
     PyObject *obj = _PyCrossInterpreterData_NewObject(data);
     if (obj == NULL) {
         assert(PyErr_Occurred());
-        (void)_release_xid_data(data, 1);
+        // It was allocated in _channel_send(), so we free it.
-        // It was allocated in _channel_send().
+        (void)_release_xid_data(data, XID_IGNORE_EXC | XID_FREE);
-        GLOBAL_FREE(data);
         return -1;
     }
-    int release_res = _release_xid_data(data, 0);
+    // It was allocated in _channel_send(), so we free it.
-    // It was allocated in _channel_send().
+    int release_res = _release_xid_data(data, XID_FREE);
-    GLOBAL_FREE(data);
     if (release_res < 0) {
         // The source interpreter has been destroyed already.
         assert(PyErr_Occurred());

Modules/_xxsubinterpretersmodule.c

@@ -53,24 +53,17 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base)
     add_new_exception(MOD, MODULE_NAME "." Py_STRINGIFY(NAME), BASE)
 
 static int
-_release_xid_data(_PyCrossInterpreterData *data, int ignoreexc)
+_release_xid_data(_PyCrossInterpreterData *data)
 {
-    PyObject *exc;
+    PyObject *exc = PyErr_GetRaisedException();
-    if (ignoreexc) {
-        exc = PyErr_GetRaisedException();
-    }
     int res = _PyCrossInterpreterData_Release(data);
     if (res < 0) {
         /* The owning interpreter is already destroyed. */
         _PyCrossInterpreterData_Clear(NULL, data);
-        if (ignoreexc) {
         // XXX Emit a warning?
         PyErr_Clear();
     }
-    }
-    if (ignoreexc) {
     PyErr_SetRaisedException(exc);
-    }
     return res;
 }
 
@@ -140,7 +133,7 @@ _sharednsitem_clear(struct _sharednsitem *item)
         PyMem_RawFree((void *)item->name);
         item->name = NULL;
     }
-    (void)_release_xid_data(&item->data, 1);
+    (void)_release_xid_data(&item->data);
 }
 
 static int
@@ -169,16 +162,16 @@ typedef struct _sharedns {
 static _sharedns *
 _sharedns_new(Py_ssize_t len)
 {
-    _sharedns *shared = PyMem_NEW(_sharedns, 1);
+    _sharedns *shared = PyMem_RawCalloc(sizeof(_sharedns), 1);
     if (shared == NULL) {
         PyErr_NoMemory();
         return NULL;
     }
     shared->len = len;
-    shared->items = PyMem_NEW(struct _sharednsitem, len);
+    shared->items = PyMem_RawCalloc(sizeof(struct _sharednsitem), len);
     if (shared->items == NULL) {
         PyErr_NoMemory();
-        PyMem_Free(shared);
+        PyMem_RawFree(shared);
         return NULL;
     }
     return shared;
@@ -190,8 +183,8 @@ _sharedns_free(_sharedns *shared)
     for (Py_ssize_t i=0; i < shared->len; i++) {
         _sharednsitem_clear(&shared->items[i]);
     }
-    PyMem_Free(shared->items);
+    PyMem_RawFree(shared->items);
-    PyMem_Free(shared);
+    PyMem_RawFree(shared);
 }
 
 static _sharedns *

Python/pystate.c

@@ -2259,10 +2259,16 @@ _xidata_init(_PyCrossInterpreterData *data)
 static inline void
 _xidata_clear(_PyCrossInterpreterData *data)
 {
+    // _PyCrossInterpreterData only has two members that need to be
+    // cleaned up, if set: "data" must be freed and "obj" must be decref'ed.
+    // In both cases the original (owning) interpreter must be used,
+    // which is the caller's responsibility to ensure.
+    if (data->data != NULL) {
         if (data->free != NULL) {
             data->free(data->data);
         }
         data->data = NULL;
+    }
     Py_CLEAR(data->obj);
 }
 
@@ -2407,40 +2413,32 @@ _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *data)
     return data->new_object(data);
 }
 
-typedef void (*releasefunc)(PyInterpreterState *, void *);
+static int
-
+_release_xidata_pending(void *data)
-static void
-_call_in_interpreter(PyInterpreterState *interp, releasefunc func, void *arg)
 {
-    /* We would use Py_AddPendingCall() if it weren't specific to the
+    _xidata_clear((_PyCrossInterpreterData *)data);
-     * main interpreter (see bpo-33608).  In the meantime we take a
+    return 0;
-     * naive approach.
-     */
-    _PyRuntimeState *runtime = interp->runtime;
-    PyThreadState *save_tstate = NULL;
-    if (interp != current_fast_get(runtime)->interp) {
-        // XXX Using the "head" thread isn't strictly correct.
-        PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
-        // XXX Possible GILState issues?
-        save_tstate = _PyThreadState_Swap(runtime, tstate);
 }
 
-    // XXX Once the GIL is per-interpreter, this should be called with the
+static int
-    // calling interpreter's GIL released and the target interpreter's held.
+_xidata_release_and_rawfree_pending(void *data)
-    func(interp, arg);
-
-    // Switch back.
-    if (save_tstate != NULL) {
-        _PyThreadState_Swap(runtime, save_tstate);
-    }
-}
-
-int
-_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
 {
-    if (data->free == NULL && data->obj == NULL) {
+    _xidata_clear((_PyCrossInterpreterData *)data);
+    PyMem_RawFree(data);
+    return 0;
+}
+
+static int
+_xidata_release(_PyCrossInterpreterData *data, int rawfree)
+{
+    if ((data->data == NULL || data->free == NULL) && data->obj == NULL) {
         // Nothing to release!
+        if (rawfree) {
+            PyMem_RawFree(data);
+        }
+        else {
             data->data = NULL;
+        }
         return 0;
     }
 
@@ -2451,15 +2449,42 @@ _PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
         // This function shouldn't have been called.
         // XXX Someone leaked some memory...
         assert(PyErr_Occurred());
+        if (rawfree) {
+            PyMem_RawFree(data);
+        }
         return -1;
     }
 
     // "Release" the data and/or the object.
-    _call_in_interpreter(interp,
+    if (interp == current_fast_get(interp->runtime)->interp) {
-                         (releasefunc)_PyCrossInterpreterData_Clear, data);
+        _xidata_clear(data);
+        if (rawfree) {
+            PyMem_RawFree(data);
+        }
+    }
+    else {
+        int (*func)(void *) = _release_xidata_pending;
+        if (rawfree) {
+            func = _xidata_release_and_rawfree_pending;
+        }
+        // XXX Emit a warning if this fails?
+        _PyEval_AddPendingCall(interp, func, data, 0);
+    }
     return 0;
 }
 
+int
+_PyCrossInterpreterData_Release(_PyCrossInterpreterData *data)
+{
+    return _xidata_release(data, 0);
+}
+
+int
+_PyCrossInterpreterData_ReleaseAndRawFree(_PyCrossInterpreterData *data)
+{
+    return _xidata_release(data, 1);
+}
+
 /* registry of {type -> crossinterpdatafunc} */
 
 /* For now we use a global registry of shareable classes.  An