gh-132776: Revert Moving memoryview XIData Code to memoryobject.c (gh-132960)

This is a partial revert of gh-132821.  It resolves the refleak introduced by that PR.

Modules/_interpretersmodule.c

@@ -9,7 +9,6 @@
 #include "pycore_code.h"          // _PyCode_HAS_EXECUTORS()
 #include "pycore_crossinterp.h"   // _PyXIData_t
 #include "pycore_interp.h"        // _PyInterpreterState_IDIncref()
-#include "pycore_memoryobject.h"  // _PyMemoryView_GetXIBuffewViewType()
 #include "pycore_modsupport.h"    // _PyArg_BadArgument()
 #include "pycore_namespace.h"     // _PyNamespace_New()
 #include "pycore_pybuffer.h"      // _PyBuffer_ReleaseInInterpreterAndRawFree()
@@ -37,6 +36,23 @@ _get_current_interp(void)
 #define look_up_interp _PyInterpreterState_LookUpIDObject
 
 
+static PyObject *
+_get_current_module(void)
+{
+    PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
+    if (name == NULL) {
+        return NULL;
+    }
+    PyObject *mod = PyImport_GetModule(name);
+    Py_DECREF(name);
+    if (mod == NULL) {
+        return NULL;
+    }
+    assert(mod != Py_None);
+    return mod;
+}
+
+
 static int
 is_running_main(PyInterpreterState *interp)
 {
@@ -55,10 +71,238 @@ is_running_main(PyInterpreterState *interp)
 }
 
 
+/* Cross-interpreter Buffer Views *******************************************/
+
+/* When a memoryview object is "shared" between interpreters,
+ * its underlying "buffer" memory is actually shared, rather than just
+ * copied.  This facilitates efficient use of that data where otherwise
+ * interpreters are strictly isolated.  However, this also means that
+ * the underlying data is subject to the complexities of thread-safety,
+ * which the user must manage carefully.
+ *
+ * When the memoryview is "shared", it is essentially copied in the same
+ * way as PyMemory_FromObject() does, but in another interpreter.
+ * The Py_buffer value is copied like normal, including the "buf" pointer,
+ * with one key exception.
+ *
+ * When a Py_buffer is released and it holds a reference to an object,
+ * that object gets a chance to call its bf_releasebuffer() (if any)
+ * before the object is decref'ed.  The same is true with the memoryview
+ * tp_dealloc, which essentially calls PyBuffer_Release().
+ *
+ * The problem for a Py_buffer shared between two interpreters is that
+ * the naive approach breaks interpreter isolation.  Operations on an
+ * object must only happen while that object's interpreter is active.
+ * If the copied mv->view.obj pointed to the original memoryview then
+ * the PyBuffer_Release() would happen under the wrong interpreter.
+ *
+ * To work around this, we set mv->view.obj on the copied memoryview
+ * to a wrapper object with the only job of releasing the original
+ * buffer in a cross-interpreter-safe way.
+ */
+
+// XXX Note that there is still an issue to sort out, where the original
+// interpreter is destroyed but code in another interpreter is still
+// using dependent buffers.  Using such buffers segfaults.  This will
+// require a careful fix.  In the meantime, users will have to be
+// diligent about avoiding the problematic situation.
+
+typedef struct {
+    PyObject base;
+    Py_buffer *view;
+    int64_t interpid;
+} xibufferview;
+
+static PyObject *
+xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid)
+{
+    assert(interpid >= 0);
+
+    Py_buffer *copied = PyMem_RawMalloc(sizeof(Py_buffer));
+    if (copied == NULL) {
+        return NULL;
+    }
+    /* This steals the view->obj reference  */
+    *copied = *view;
+
+    xibufferview *self = PyObject_Malloc(sizeof(xibufferview));
+    if (self == NULL) {
+        PyMem_RawFree(copied);
+        return NULL;
+    }
+    PyObject_Init(&self->base, cls);
+    *self = (xibufferview){
+        .base = self->base,
+        .view = copied,
+        .interpid = interpid,
+    };
+    return (PyObject *)self;
+}
+
+static void
+xibufferview_dealloc(PyObject *op)
+{
+    xibufferview *self = (xibufferview *)op;
+    if (self->view != NULL) {
+        PyInterpreterState *interp =
+                        _PyInterpreterState_LookUpID(self->interpid);
+        if (interp == NULL) {
+            /* The interpreter is no longer alive. */
+            PyErr_Clear();
+            PyMem_RawFree(self->view);
+        }
+        else {
+            if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp,
+                                                         self->view) < 0)
+            {
+                // XXX Emit a warning?
+                PyErr_Clear();
+            }
+        }
+    }
+
+    PyTypeObject *tp = Py_TYPE(self);
+    tp->tp_free(self);
+    /* "Instances of heap-allocated types hold a reference to their type."
+     * See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
+     * See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
+    */
+    // XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
+    // like we do for _abc._abc_data?
+    Py_DECREF(tp);
+}
+
+static int
+xibufferview_getbuf(PyObject *op, Py_buffer *view, int flags)
+{
+    /* Only PyMemoryView_FromObject() should ever call this,
+       via _memoryview_from_xid() below. */
+    xibufferview *self = (xibufferview *)op;
+    *view = *self->view;
+    /* This is the workaround mentioned earlier. */
+    view->obj = op;
+    // XXX Should we leave it alone?
+    view->internal = NULL;
+    return 0;
+}
+
+static PyType_Slot XIBufferViewType_slots[] = {
+    {Py_tp_dealloc, xibufferview_dealloc},
+    {Py_bf_getbuffer, xibufferview_getbuf},
+    // We don't bother with Py_bf_releasebuffer since we don't need it.
+    {0, NULL},
+};
+
+static PyType_Spec XIBufferViewType_spec = {
+    .name = MODULE_NAME_STR ".CrossInterpreterBufferView",
+    .basicsize = sizeof(xibufferview),
+    .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
+              Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
+    .slots = XIBufferViewType_slots,
+};
+
+
+static PyTypeObject * _get_current_xibufferview_type(void);
+
+
+struct xibuffer {
+    Py_buffer view;
+    int used;
+};
+
+static PyObject *
+_memoryview_from_xid(_PyXIData_t *data)
+{
+    assert(_PyXIData_DATA(data) != NULL);
+    assert(_PyXIData_OBJ(data) == NULL);
+    assert(_PyXIData_INTERPID(data) >= 0);
+    struct xibuffer *view = (struct xibuffer *)_PyXIData_DATA(data);
+    assert(!view->used);
+
+    PyTypeObject *cls = _get_current_xibufferview_type();
+    if (cls == NULL) {
+        return NULL;
+    }
+
+    PyObject *obj = xibufferview_from_buffer(
+                        cls, &view->view, _PyXIData_INTERPID(data));
+    if (obj == NULL) {
+        return NULL;
+    }
+    PyObject *res = PyMemoryView_FromObject(obj);
+    if (res == NULL) {
+        Py_DECREF(obj);
+        return NULL;
+    }
+    view->used = 1;
+    return res;
+}
+
+static void
+_pybuffer_shared_free(void* data)
+{
+    struct xibuffer *view = (struct xibuffer *)data;
+    if (!view->used) {
+        PyBuffer_Release(&view->view);
+    }
+    PyMem_RawFree(data);
+}
+
+static int
+_pybuffer_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data)
+{
+    struct xibuffer *view = PyMem_RawMalloc(sizeof(struct xibuffer));
+    if (view == NULL) {
+        return -1;
+    }
+    view->used = 0;
+    /* This will increment the memoryview's export count, which won't get
+     * decremented until the view sent to other interpreters is released. */
+    if (PyObject_GetBuffer(obj, &view->view, PyBUF_FULL_RO) < 0) {
+        PyMem_RawFree(view);
+        return -1;
+    }
+    /* The view holds a reference to the object, so we don't worry
+     * about also tracking it on the cross-interpreter data. */
+    _PyXIData_Init(data, tstate->interp, view, NULL, _memoryview_from_xid);
+    data->free = _pybuffer_shared_free;
+    return 0;
+}
+
+static int
+register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
+{
+    // XIBufferView
+    assert(*p_state == NULL);
+    PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec(
+                mod, &XIBufferViewType_spec, NULL);
+    if (cls == NULL) {
+        return -1;
+    }
+    if (PyModule_AddType(mod, cls) < 0) {
+        Py_DECREF(cls);
+        return -1;
+    }
+    *p_state = cls;
+
+    // Register XID for the builtin memoryview type.
+    if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) {
+        return -1;
+    }
+    // We don't ever bother un-registering memoryview.
+
+    return 0;
+}
+
+
+
 /* module state *************************************************************/
 
 typedef struct {
     int _notused;
+
+    /* heap types */
+    PyTypeObject *XIBufferViewType;
 } module_state;
 
 static inline module_state *
@@ -70,19 +314,51 @@ get_module_state(PyObject *mod)
     return state;
 }
 
+static module_state *
+_get_current_module_state(void)
+{
+    PyObject *mod = _get_current_module();
+    if (mod == NULL) {
+        // XXX import it?
+        PyErr_SetString(PyExc_RuntimeError,
+                        MODULE_NAME_STR " module not imported yet");
+        return NULL;
+    }
+    module_state *state = get_module_state(mod);
+    Py_DECREF(mod);
+    return state;
+}
+
 static int
 traverse_module_state(module_state *state, visitproc visit, void *arg)
 {
+    /* heap types */
+    Py_VISIT(state->XIBufferViewType);
+
     return 0;
 }
 
 static int
 clear_module_state(module_state *state)
 {
+    /* heap types */
+    Py_CLEAR(state->XIBufferViewType);
+
     return 0;
 }
 
 
+static PyTypeObject *
+_get_current_xibufferview_type(void)
+{
+    module_state *state = _get_current_module_state();
+    if (state == NULL) {
+        return NULL;
+    }
+    return state->XIBufferViewType;
+}
+
+
 /* Python code **************************************************************/
 
 static const char *
@@ -1303,7 +1579,6 @@ module_exec(PyObject *mod)
 {
     PyInterpreterState *interp = PyInterpreterState_Get();
     module_state *state = get_module_state(mod);
-    (void)state;
 
     _PyXIData_lookup_context_t ctx;
     if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
@@ -1335,14 +1610,9 @@ module_exec(PyObject *mod)
         goto error;
     }
 
-    PyTypeObject *XIBufferViewType = _PyMemoryView_GetXIBuffewViewType();
-    if (XIBufferViewType == NULL) {
+    if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) {
         goto error;
     }
-    if (PyModule_AddType(mod, XIBufferViewType) < 0) {
-        Py_DECREF(XIBufferViewType);
-        return -1;
-    }
 
     return 0;