mirror of
https://github.com/python/cpython.git
synced 2025-07-07 19:35:27 +00:00
gh-132776: Revert Moving memoryview XIData Code to memoryobject.c (gh-132960)
This is a partial revert of gh-132821. It resolves the refleak introduced by that PR.
This commit is contained in:
parent
622176513e
commit
2a28b21a51
8 changed files with 284 additions and 339 deletions
|
@ -9,7 +9,6 @@ extern "C" {
|
||||||
|
|
||||||
#include "pycore_ast_state.h" // struct ast_state
|
#include "pycore_ast_state.h" // struct ast_state
|
||||||
#include "pycore_llist.h" // struct llist_node
|
#include "pycore_llist.h" // struct llist_node
|
||||||
#include "pycore_memoryobject.h" // struct _memoryobject_state
|
|
||||||
#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS
|
#include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS
|
||||||
#include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR
|
#include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR
|
||||||
#include "pycore_structs.h" // PyHamtObject
|
#include "pycore_structs.h" // PyHamtObject
|
||||||
|
@ -913,10 +912,9 @@ struct _is {
|
||||||
struct _dtoa_state dtoa;
|
struct _dtoa_state dtoa;
|
||||||
struct _py_func_state func_state;
|
struct _py_func_state func_state;
|
||||||
struct _py_code_state code_state;
|
struct _py_code_state code_state;
|
||||||
|
|
||||||
struct _Py_dict_state dict_state;
|
struct _Py_dict_state dict_state;
|
||||||
struct _Py_exc_state exc_state;
|
struct _Py_exc_state exc_state;
|
||||||
struct _memoryobject_state memobj_state;
|
|
||||||
|
|
||||||
struct _Py_mem_interp_free_queue mem_free_queue;
|
struct _Py_mem_interp_free_queue mem_free_queue;
|
||||||
|
|
||||||
struct ast_state ast;
|
struct ast_state ast;
|
||||||
|
|
|
@ -8,17 +8,6 @@ extern "C" {
|
||||||
# error "this header requires Py_BUILD_CORE define"
|
# error "this header requires Py_BUILD_CORE define"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
struct _memoryobject_state {
|
|
||||||
PyTypeObject *XIBufferViewType;
|
|
||||||
};
|
|
||||||
|
|
||||||
extern PyStatus _PyMemoryView_InitTypes(PyInterpreterState *);
|
|
||||||
extern void _PyMemoryView_FiniTypes(PyInterpreterState *);
|
|
||||||
|
|
||||||
// exported for _interpreters module
|
|
||||||
PyAPI_FUNC(PyTypeObject *) _PyMemoryView_GetXIBuffewViewType(void);
|
|
||||||
|
|
||||||
|
|
||||||
extern PyTypeObject _PyManagedBuffer_Type;
|
extern PyTypeObject _PyManagedBuffer_Type;
|
||||||
|
|
||||||
PyObject *
|
PyObject *
|
||||||
|
|
|
@ -5,7 +5,6 @@
|
||||||
_RESOLVE_MODINIT_FUNC_NAME(NAME)
|
_RESOLVE_MODINIT_FUNC_NAME(NAME)
|
||||||
|
|
||||||
|
|
||||||
#ifdef REGISTERS_HEAP_TYPES
|
|
||||||
static int
|
static int
|
||||||
ensure_xid_class(PyTypeObject *cls, xidatafunc getdata)
|
ensure_xid_class(PyTypeObject *cls, xidatafunc getdata)
|
||||||
{
|
{
|
||||||
|
@ -17,6 +16,7 @@ ensure_xid_class(PyTypeObject *cls, xidatafunc getdata)
|
||||||
return _PyXIData_RegisterClass(&ctx, cls, getdata);
|
return _PyXIData_RegisterClass(&ctx, cls, getdata);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#ifdef REGISTERS_HEAP_TYPES
|
||||||
static int
|
static int
|
||||||
clear_xid_class(PyTypeObject *cls)
|
clear_xid_class(PyTypeObject *cls)
|
||||||
{
|
{
|
||||||
|
|
|
@ -9,7 +9,6 @@
|
||||||
#include "pycore_code.h" // _PyCode_HAS_EXECUTORS()
|
#include "pycore_code.h" // _PyCode_HAS_EXECUTORS()
|
||||||
#include "pycore_crossinterp.h" // _PyXIData_t
|
#include "pycore_crossinterp.h" // _PyXIData_t
|
||||||
#include "pycore_interp.h" // _PyInterpreterState_IDIncref()
|
#include "pycore_interp.h" // _PyInterpreterState_IDIncref()
|
||||||
#include "pycore_memoryobject.h" // _PyMemoryView_GetXIBuffewViewType()
|
|
||||||
#include "pycore_modsupport.h" // _PyArg_BadArgument()
|
#include "pycore_modsupport.h" // _PyArg_BadArgument()
|
||||||
#include "pycore_namespace.h" // _PyNamespace_New()
|
#include "pycore_namespace.h" // _PyNamespace_New()
|
||||||
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
|
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
|
||||||
|
@ -37,6 +36,23 @@ _get_current_interp(void)
|
||||||
#define look_up_interp _PyInterpreterState_LookUpIDObject
|
#define look_up_interp _PyInterpreterState_LookUpIDObject
|
||||||
|
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
_get_current_module(void)
|
||||||
|
{
|
||||||
|
PyObject *name = PyUnicode_FromString(MODULE_NAME_STR);
|
||||||
|
if (name == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
PyObject *mod = PyImport_GetModule(name);
|
||||||
|
Py_DECREF(name);
|
||||||
|
if (mod == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
assert(mod != Py_None);
|
||||||
|
return mod;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static int
|
static int
|
||||||
is_running_main(PyInterpreterState *interp)
|
is_running_main(PyInterpreterState *interp)
|
||||||
{
|
{
|
||||||
|
@ -55,10 +71,238 @@ is_running_main(PyInterpreterState *interp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/* Cross-interpreter Buffer Views *******************************************/
|
||||||
|
|
||||||
|
/* When a memoryview object is "shared" between interpreters,
|
||||||
|
* its underlying "buffer" memory is actually shared, rather than just
|
||||||
|
* copied. This facilitates efficient use of that data where otherwise
|
||||||
|
* interpreters are strictly isolated. However, this also means that
|
||||||
|
* the underlying data is subject to the complexities of thread-safety,
|
||||||
|
* which the user must manage carefully.
|
||||||
|
*
|
||||||
|
* When the memoryview is "shared", it is essentially copied in the same
|
||||||
|
* way as PyMemory_FromObject() does, but in another interpreter.
|
||||||
|
* The Py_buffer value is copied like normal, including the "buf" pointer,
|
||||||
|
* with one key exception.
|
||||||
|
*
|
||||||
|
* When a Py_buffer is released and it holds a reference to an object,
|
||||||
|
* that object gets a chance to call its bf_releasebuffer() (if any)
|
||||||
|
* before the object is decref'ed. The same is true with the memoryview
|
||||||
|
* tp_dealloc, which essentially calls PyBuffer_Release().
|
||||||
|
*
|
||||||
|
* The problem for a Py_buffer shared between two interpreters is that
|
||||||
|
* the naive approach breaks interpreter isolation. Operations on an
|
||||||
|
* object must only happen while that object's interpreter is active.
|
||||||
|
* If the copied mv->view.obj pointed to the original memoryview then
|
||||||
|
* the PyBuffer_Release() would happen under the wrong interpreter.
|
||||||
|
*
|
||||||
|
* To work around this, we set mv->view.obj on the copied memoryview
|
||||||
|
* to a wrapper object with the only job of releasing the original
|
||||||
|
* buffer in a cross-interpreter-safe way.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// XXX Note that there is still an issue to sort out, where the original
|
||||||
|
// interpreter is destroyed but code in another interpreter is still
|
||||||
|
// using dependent buffers. Using such buffers segfaults. This will
|
||||||
|
// require a careful fix. In the meantime, users will have to be
|
||||||
|
// diligent about avoiding the problematic situation.
|
||||||
|
|
||||||
|
typedef struct {
|
||||||
|
PyObject base;
|
||||||
|
Py_buffer *view;
|
||||||
|
int64_t interpid;
|
||||||
|
} xibufferview;
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid)
|
||||||
|
{
|
||||||
|
assert(interpid >= 0);
|
||||||
|
|
||||||
|
Py_buffer *copied = PyMem_RawMalloc(sizeof(Py_buffer));
|
||||||
|
if (copied == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
/* This steals the view->obj reference */
|
||||||
|
*copied = *view;
|
||||||
|
|
||||||
|
xibufferview *self = PyObject_Malloc(sizeof(xibufferview));
|
||||||
|
if (self == NULL) {
|
||||||
|
PyMem_RawFree(copied);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
PyObject_Init(&self->base, cls);
|
||||||
|
*self = (xibufferview){
|
||||||
|
.base = self->base,
|
||||||
|
.view = copied,
|
||||||
|
.interpid = interpid,
|
||||||
|
};
|
||||||
|
return (PyObject *)self;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
xibufferview_dealloc(PyObject *op)
|
||||||
|
{
|
||||||
|
xibufferview *self = (xibufferview *)op;
|
||||||
|
if (self->view != NULL) {
|
||||||
|
PyInterpreterState *interp =
|
||||||
|
_PyInterpreterState_LookUpID(self->interpid);
|
||||||
|
if (interp == NULL) {
|
||||||
|
/* The interpreter is no longer alive. */
|
||||||
|
PyErr_Clear();
|
||||||
|
PyMem_RawFree(self->view);
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp,
|
||||||
|
self->view) < 0)
|
||||||
|
{
|
||||||
|
// XXX Emit a warning?
|
||||||
|
PyErr_Clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
PyTypeObject *tp = Py_TYPE(self);
|
||||||
|
tp->tp_free(self);
|
||||||
|
/* "Instances of heap-allocated types hold a reference to their type."
|
||||||
|
* See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
|
||||||
|
* See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
|
||||||
|
*/
|
||||||
|
// XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
|
||||||
|
// like we do for _abc._abc_data?
|
||||||
|
Py_DECREF(tp);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
xibufferview_getbuf(PyObject *op, Py_buffer *view, int flags)
|
||||||
|
{
|
||||||
|
/* Only PyMemoryView_FromObject() should ever call this,
|
||||||
|
via _memoryview_from_xid() below. */
|
||||||
|
xibufferview *self = (xibufferview *)op;
|
||||||
|
*view = *self->view;
|
||||||
|
/* This is the workaround mentioned earlier. */
|
||||||
|
view->obj = op;
|
||||||
|
// XXX Should we leave it alone?
|
||||||
|
view->internal = NULL;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static PyType_Slot XIBufferViewType_slots[] = {
|
||||||
|
{Py_tp_dealloc, xibufferview_dealloc},
|
||||||
|
{Py_bf_getbuffer, xibufferview_getbuf},
|
||||||
|
// We don't bother with Py_bf_releasebuffer since we don't need it.
|
||||||
|
{0, NULL},
|
||||||
|
};
|
||||||
|
|
||||||
|
static PyType_Spec XIBufferViewType_spec = {
|
||||||
|
.name = MODULE_NAME_STR ".CrossInterpreterBufferView",
|
||||||
|
.basicsize = sizeof(xibufferview),
|
||||||
|
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
|
||||||
|
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
|
||||||
|
.slots = XIBufferViewType_slots,
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
static PyTypeObject * _get_current_xibufferview_type(void);
|
||||||
|
|
||||||
|
|
||||||
|
struct xibuffer {
|
||||||
|
Py_buffer view;
|
||||||
|
int used;
|
||||||
|
};
|
||||||
|
|
||||||
|
static PyObject *
|
||||||
|
_memoryview_from_xid(_PyXIData_t *data)
|
||||||
|
{
|
||||||
|
assert(_PyXIData_DATA(data) != NULL);
|
||||||
|
assert(_PyXIData_OBJ(data) == NULL);
|
||||||
|
assert(_PyXIData_INTERPID(data) >= 0);
|
||||||
|
struct xibuffer *view = (struct xibuffer *)_PyXIData_DATA(data);
|
||||||
|
assert(!view->used);
|
||||||
|
|
||||||
|
PyTypeObject *cls = _get_current_xibufferview_type();
|
||||||
|
if (cls == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
PyObject *obj = xibufferview_from_buffer(
|
||||||
|
cls, &view->view, _PyXIData_INTERPID(data));
|
||||||
|
if (obj == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
PyObject *res = PyMemoryView_FromObject(obj);
|
||||||
|
if (res == NULL) {
|
||||||
|
Py_DECREF(obj);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
view->used = 1;
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
static void
|
||||||
|
_pybuffer_shared_free(void* data)
|
||||||
|
{
|
||||||
|
struct xibuffer *view = (struct xibuffer *)data;
|
||||||
|
if (!view->used) {
|
||||||
|
PyBuffer_Release(&view->view);
|
||||||
|
}
|
||||||
|
PyMem_RawFree(data);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
_pybuffer_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data)
|
||||||
|
{
|
||||||
|
struct xibuffer *view = PyMem_RawMalloc(sizeof(struct xibuffer));
|
||||||
|
if (view == NULL) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
view->used = 0;
|
||||||
|
/* This will increment the memoryview's export count, which won't get
|
||||||
|
* decremented until the view sent to other interpreters is released. */
|
||||||
|
if (PyObject_GetBuffer(obj, &view->view, PyBUF_FULL_RO) < 0) {
|
||||||
|
PyMem_RawFree(view);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
/* The view holds a reference to the object, so we don't worry
|
||||||
|
* about also tracking it on the cross-interpreter data. */
|
||||||
|
_PyXIData_Init(data, tstate->interp, view, NULL, _memoryview_from_xid);
|
||||||
|
data->free = _pybuffer_shared_free;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int
|
||||||
|
register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
|
||||||
|
{
|
||||||
|
// XIBufferView
|
||||||
|
assert(*p_state == NULL);
|
||||||
|
PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec(
|
||||||
|
mod, &XIBufferViewType_spec, NULL);
|
||||||
|
if (cls == NULL) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
if (PyModule_AddType(mod, cls) < 0) {
|
||||||
|
Py_DECREF(cls);
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
*p_state = cls;
|
||||||
|
|
||||||
|
// Register XID for the builtin memoryview type.
|
||||||
|
if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) {
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
// We don't ever bother un-registering memoryview.
|
||||||
|
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
/* module state *************************************************************/
|
/* module state *************************************************************/
|
||||||
|
|
||||||
typedef struct {
|
typedef struct {
|
||||||
int _notused;
|
int _notused;
|
||||||
|
|
||||||
|
/* heap types */
|
||||||
|
PyTypeObject *XIBufferViewType;
|
||||||
} module_state;
|
} module_state;
|
||||||
|
|
||||||
static inline module_state *
|
static inline module_state *
|
||||||
|
@ -70,19 +314,51 @@ get_module_state(PyObject *mod)
|
||||||
return state;
|
return state;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static module_state *
|
||||||
|
_get_current_module_state(void)
|
||||||
|
{
|
||||||
|
PyObject *mod = _get_current_module();
|
||||||
|
if (mod == NULL) {
|
||||||
|
// XXX import it?
|
||||||
|
PyErr_SetString(PyExc_RuntimeError,
|
||||||
|
MODULE_NAME_STR " module not imported yet");
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
module_state *state = get_module_state(mod);
|
||||||
|
Py_DECREF(mod);
|
||||||
|
return state;
|
||||||
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
traverse_module_state(module_state *state, visitproc visit, void *arg)
|
traverse_module_state(module_state *state, visitproc visit, void *arg)
|
||||||
{
|
{
|
||||||
|
/* heap types */
|
||||||
|
Py_VISIT(state->XIBufferViewType);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
static int
|
static int
|
||||||
clear_module_state(module_state *state)
|
clear_module_state(module_state *state)
|
||||||
{
|
{
|
||||||
|
/* heap types */
|
||||||
|
Py_CLEAR(state->XIBufferViewType);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
static PyTypeObject *
|
||||||
|
_get_current_xibufferview_type(void)
|
||||||
|
{
|
||||||
|
module_state *state = _get_current_module_state();
|
||||||
|
if (state == NULL) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
return state->XIBufferViewType;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
/* Python code **************************************************************/
|
/* Python code **************************************************************/
|
||||||
|
|
||||||
static const char *
|
static const char *
|
||||||
|
@ -1303,7 +1579,6 @@ module_exec(PyObject *mod)
|
||||||
{
|
{
|
||||||
PyInterpreterState *interp = PyInterpreterState_Get();
|
PyInterpreterState *interp = PyInterpreterState_Get();
|
||||||
module_state *state = get_module_state(mod);
|
module_state *state = get_module_state(mod);
|
||||||
(void)state;
|
|
||||||
|
|
||||||
_PyXIData_lookup_context_t ctx;
|
_PyXIData_lookup_context_t ctx;
|
||||||
if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
|
if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
|
||||||
|
@ -1335,14 +1610,9 @@ module_exec(PyObject *mod)
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
|
|
||||||
PyTypeObject *XIBufferViewType = _PyMemoryView_GetXIBuffewViewType();
|
if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) {
|
||||||
if (XIBufferViewType == NULL) {
|
|
||||||
goto error;
|
goto error;
|
||||||
}
|
}
|
||||||
if (PyModule_AddType(mod, XIBufferViewType) < 0) {
|
|
||||||
Py_DECREF(XIBufferViewType);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
|
|
@ -12,11 +12,8 @@
|
||||||
|
|
||||||
#include "Python.h"
|
#include "Python.h"
|
||||||
#include "pycore_abstract.h" // _PyIndex_Check()
|
#include "pycore_abstract.h" // _PyIndex_Check()
|
||||||
#include "pycore_initconfig.h" // _PyStatus_OK()
|
|
||||||
#include "pycore_interp.h" // _PyInterpreterState_LookUpID()
|
|
||||||
#include "pycore_memoryobject.h" // _PyManagedBuffer_Type
|
#include "pycore_memoryobject.h" // _PyManagedBuffer_Type
|
||||||
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
|
#include "pycore_object.h" // _PyObject_GC_UNTRACK()
|
||||||
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
|
|
||||||
#include "pycore_strhex.h" // _Py_strhex_with_sep()
|
#include "pycore_strhex.h" // _Py_strhex_with_sep()
|
||||||
#include <stddef.h> // offsetof()
|
#include <stddef.h> // offsetof()
|
||||||
|
|
||||||
|
@ -3559,252 +3556,6 @@ PyTypeObject _PyMemoryIter_Type = {
|
||||||
.tp_iternext = memoryiter_next,
|
.tp_iternext = memoryiter_next,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/**************************************************************************/
|
|
||||||
/* Memoryview Cross-interpreter Data */
|
|
||||||
/**************************************************************************/
|
|
||||||
|
|
||||||
/* When a memoryview object is "shared" between interpreters,
|
|
||||||
* its underlying "buffer" memory is actually shared, rather than just
|
|
||||||
* copied. This facilitates efficient use of that data where otherwise
|
|
||||||
* interpreters are strictly isolated. However, this also means that
|
|
||||||
* the underlying data is subject to the complexities of thread-safety,
|
|
||||||
* which the user must manage carefully.
|
|
||||||
*
|
|
||||||
* When the memoryview is "shared", it is essentially copied in the same
|
|
||||||
* way as PyMemory_FromObject() does, but in another interpreter.
|
|
||||||
* The Py_buffer value is copied like normal, including the "buf" pointer,
|
|
||||||
* with one key exception.
|
|
||||||
*
|
|
||||||
* When a Py_buffer is released and it holds a reference to an object,
|
|
||||||
* that object gets a chance to call its bf_releasebuffer() (if any)
|
|
||||||
* before the object is decref'ed. The same is true with the memoryview
|
|
||||||
* tp_dealloc, which essentially calls PyBuffer_Release().
|
|
||||||
*
|
|
||||||
* The problem for a Py_buffer shared between two interpreters is that
|
|
||||||
* the naive approach breaks interpreter isolation. Operations on an
|
|
||||||
* object must only happen while that object's interpreter is active.
|
|
||||||
* If the copied mv->view.obj pointed to the original memoryview then
|
|
||||||
* the PyBuffer_Release() would happen under the wrong interpreter.
|
|
||||||
*
|
|
||||||
* To work around this, we set mv->view.obj on the copied memoryview
|
|
||||||
* to a wrapper object with the only job of releasing the original
|
|
||||||
* buffer in a cross-interpreter-safe way.
|
|
||||||
*/
|
|
||||||
|
|
||||||
// XXX Note that there is still an issue to sort out, where the original
|
|
||||||
// interpreter is destroyed but code in another interpreter is still
|
|
||||||
// using dependent buffers. Using such buffers segfaults. This will
|
|
||||||
// require a careful fix. In the meantime, users will have to be
|
|
||||||
// diligent about avoiding the problematic situation.
|
|
||||||
|
|
||||||
typedef struct {
|
|
||||||
PyObject base;
|
|
||||||
Py_buffer *view;
|
|
||||||
int64_t interpid;
|
|
||||||
} xibufferview;
|
|
||||||
|
|
||||||
static PyObject *
|
|
||||||
xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid)
|
|
||||||
{
|
|
||||||
assert(interpid >= 0);
|
|
||||||
|
|
||||||
Py_buffer *copied = PyMem_RawMalloc(sizeof(Py_buffer));
|
|
||||||
if (copied == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
/* This steals the view->obj reference */
|
|
||||||
*copied = *view;
|
|
||||||
|
|
||||||
xibufferview *self = PyObject_Malloc(sizeof(xibufferview));
|
|
||||||
if (self == NULL) {
|
|
||||||
PyMem_RawFree(copied);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
*self = (xibufferview){
|
|
||||||
.view = copied,
|
|
||||||
.interpid = interpid,
|
|
||||||
};
|
|
||||||
PyObject_Init(&self->base, cls);
|
|
||||||
return (PyObject *)self;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
xibufferview_dealloc(PyObject *op)
|
|
||||||
{
|
|
||||||
xibufferview *self = (xibufferview *)op;
|
|
||||||
|
|
||||||
if (self->view != NULL) {
|
|
||||||
PyInterpreterState *interp =
|
|
||||||
_PyInterpreterState_LookUpID(self->interpid);
|
|
||||||
if (interp == NULL) {
|
|
||||||
/* The interpreter is no longer alive. */
|
|
||||||
PyErr_Clear();
|
|
||||||
PyMem_RawFree(self->view);
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp,
|
|
||||||
self->view) < 0)
|
|
||||||
{
|
|
||||||
// XXX Emit a warning?
|
|
||||||
PyErr_Clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
PyTypeObject *tp = Py_TYPE(self);
|
|
||||||
tp->tp_free(self);
|
|
||||||
/* "Instances of heap-allocated types hold a reference to their type."
|
|
||||||
* See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
|
|
||||||
* See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
|
|
||||||
*/
|
|
||||||
// XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
|
|
||||||
// like we do for _abc._abc_data?
|
|
||||||
Py_DECREF(tp);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
|
||||||
xibufferview_getbuf(PyObject *op, Py_buffer *view, int flags)
|
|
||||||
{
|
|
||||||
/* Only PyMemoryView_FromObject() should ever call this,
|
|
||||||
via _memoryview_from_xid() below. */
|
|
||||||
xibufferview *self = (xibufferview *)op;
|
|
||||||
*view = *self->view;
|
|
||||||
/* This is the workaround mentioned earlier. */
|
|
||||||
view->obj = op;
|
|
||||||
// XXX Should we leave it alone?
|
|
||||||
view->internal = NULL;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static PyType_Slot XIBufferViewType_slots[] = {
|
|
||||||
{Py_tp_dealloc, xibufferview_dealloc},
|
|
||||||
{Py_bf_getbuffer, xibufferview_getbuf},
|
|
||||||
// We don't bother with Py_bf_releasebuffer since we don't need it.
|
|
||||||
{0, NULL},
|
|
||||||
};
|
|
||||||
|
|
||||||
static PyType_Spec XIBufferViewType_spec = {
|
|
||||||
.name = "_interpreters.CrossInterpreterBufferView",
|
|
||||||
.basicsize = sizeof(xibufferview),
|
|
||||||
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
|
|
||||||
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
|
|
||||||
.slots = XIBufferViewType_slots,
|
|
||||||
};
|
|
||||||
|
|
||||||
PyTypeObject *
|
|
||||||
_PyMemoryView_GetXIBuffewViewType(void)
|
|
||||||
{
|
|
||||||
PyInterpreterState *interp = _PyInterpreterState_GET();
|
|
||||||
PyTypeObject *cls = interp->memobj_state.XIBufferViewType;
|
|
||||||
if (cls == NULL) {
|
|
||||||
cls = (PyTypeObject *)PyType_FromSpec(&XIBufferViewType_spec);
|
|
||||||
if (cls == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
/* It gets cleaned up during interpreter finalization
|
|
||||||
* in clear_xidata_state(). */
|
|
||||||
interp->memobj_state.XIBufferViewType = cls;
|
|
||||||
}
|
|
||||||
Py_INCREF(cls);
|
|
||||||
return cls;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
struct xibuffer {
|
|
||||||
Py_buffer view;
|
|
||||||
int used;
|
|
||||||
};
|
|
||||||
|
|
||||||
static PyObject *
|
|
||||||
_memoryview_from_xid(_PyXIData_t *data)
|
|
||||||
{
|
|
||||||
assert(_PyXIData_DATA(data) != NULL);
|
|
||||||
assert(_PyXIData_OBJ(data) == NULL);
|
|
||||||
assert(_PyXIData_INTERPID(data) >= 0);
|
|
||||||
struct xibuffer *view = (struct xibuffer *)_PyXIData_DATA(data);
|
|
||||||
assert(!view->used);
|
|
||||||
|
|
||||||
PyTypeObject *cls = _PyMemoryView_GetXIBuffewViewType();
|
|
||||||
if (cls == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
PyObject *obj = xibufferview_from_buffer(
|
|
||||||
cls, &view->view, _PyXIData_INTERPID(data));
|
|
||||||
if (obj == NULL) {
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
PyObject *res = PyMemoryView_FromObject(obj);
|
|
||||||
if (res == NULL) {
|
|
||||||
Py_DECREF(obj);
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
view->used = 1;
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
_pybuffer_shared_free(void* data)
|
|
||||||
{
|
|
||||||
struct xibuffer *view = (struct xibuffer *)data;
|
|
||||||
if (!view->used) {
|
|
||||||
PyBuffer_Release(&view->view);
|
|
||||||
}
|
|
||||||
PyMem_RawFree(data);
|
|
||||||
}
|
|
||||||
|
|
||||||
static int
|
|
||||||
_pybuffer_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data)
|
|
||||||
{
|
|
||||||
struct xibuffer *view = PyMem_RawMalloc(sizeof(struct xibuffer));
|
|
||||||
if (view == NULL) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
view->used = 0;
|
|
||||||
/* This will increment the memoryview's export count, which won't get
|
|
||||||
* decremented until the view sent to other interpreters is released. */
|
|
||||||
if (PyObject_GetBuffer(obj, &view->view, PyBUF_FULL_RO) < 0) {
|
|
||||||
PyMem_RawFree(view);
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
/* The view holds a reference to the object, so we don't worry
|
|
||||||
* about also tracking it on the cross-interpreter data. */
|
|
||||||
_PyXIData_Init(data, tstate->interp, view, NULL, _memoryview_from_xid);
|
|
||||||
data->free = _pybuffer_shared_free;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
static int
|
|
||||||
init_xidata_types(PyInterpreterState *interp)
|
|
||||||
{
|
|
||||||
/* Register an XI data handler for memoryview. */
|
|
||||||
// This is necessary only as long as we don't have a tp_ slot for it.
|
|
||||||
_PyXIData_lookup_context_t ctx;
|
|
||||||
if (_PyXIData_GetLookupContext(interp, &ctx) < 0) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
if (_PyXIData_RegisterClass(&ctx, &PyMemoryView_Type, _pybuffer_shared) < 0) {
|
|
||||||
return -1;
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void
|
|
||||||
clear_xidata_types(PyInterpreterState *interp)
|
|
||||||
{
|
|
||||||
if (interp->memobj_state.XIBufferViewType != NULL) {
|
|
||||||
Py_CLEAR(interp->memobj_state.XIBufferViewType);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/**************************************************************************/
|
|
||||||
/* Memoryview Type */
|
|
||||||
/**************************************************************************/
|
|
||||||
|
|
||||||
PyTypeObject PyMemoryView_Type = {
|
PyTypeObject PyMemoryView_Type = {
|
||||||
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
PyVarObject_HEAD_INIT(&PyType_Type, 0)
|
||||||
"memoryview", /* tp_name */
|
"memoryview", /* tp_name */
|
||||||
|
@ -3846,27 +3597,3 @@ PyTypeObject PyMemoryView_Type = {
|
||||||
0, /* tp_alloc */
|
0, /* tp_alloc */
|
||||||
memoryview, /* tp_new */
|
memoryview, /* tp_new */
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/**************************************************************************/
|
|
||||||
/* Runtime Lifecycle */
|
|
||||||
/**************************************************************************/
|
|
||||||
|
|
||||||
PyStatus
|
|
||||||
_PyMemoryView_InitTypes(PyInterpreterState *interp)
|
|
||||||
{
|
|
||||||
/* interp->memobj_state.XIBufferViewType is initialized lazily
|
|
||||||
* in _PyMemoryView_GetXIBuffewViewType(). */
|
|
||||||
|
|
||||||
if (init_xidata_types(interp) < 0) {
|
|
||||||
return _PyStatus_ERR("failed to initialize cross-interpreter data types");
|
|
||||||
}
|
|
||||||
|
|
||||||
return _PyStatus_OK();
|
|
||||||
}
|
|
||||||
|
|
||||||
void
|
|
||||||
_PyMemoryView_FiniTypes(PyInterpreterState *interp)
|
|
||||||
{
|
|
||||||
clear_xidata_types(interp);
|
|
||||||
}
|
|
||||||
|
|
|
@ -1902,25 +1902,7 @@ _PyXI_Fini(PyInterpreterState *interp)
|
||||||
PyStatus
|
PyStatus
|
||||||
_PyXI_InitTypes(PyInterpreterState *interp)
|
_PyXI_InitTypes(PyInterpreterState *interp)
|
||||||
{
|
{
|
||||||
if (_Py_IsMainInterpreter(interp)) {
|
if (init_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp) < 0) {
|
||||||
_PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp);
|
|
||||||
if (global_state == NULL) {
|
|
||||||
PyErr_PrintEx(0);
|
|
||||||
return _PyStatus_ERR(
|
|
||||||
"failed to get global cross-interpreter state");
|
|
||||||
}
|
|
||||||
xid_lookup_init(&global_state->data_lookup);
|
|
||||||
}
|
|
||||||
|
|
||||||
_PyXI_state_t *state = _PyXI_GET_STATE(interp);
|
|
||||||
if (state == NULL) {
|
|
||||||
PyErr_PrintEx(0);
|
|
||||||
return _PyStatus_ERR(
|
|
||||||
"failed to get interpreter's cross-interpreter state");
|
|
||||||
}
|
|
||||||
xid_lookup_init(&state->data_lookup);
|
|
||||||
|
|
||||||
if (init_static_exctypes(&state->exceptions, interp) < 0) {
|
|
||||||
PyErr_PrintEx(0);
|
PyErr_PrintEx(0);
|
||||||
return _PyStatus_ERR(
|
return _PyStatus_ERR(
|
||||||
"failed to initialize the cross-interpreter exception types");
|
"failed to initialize the cross-interpreter exception types");
|
||||||
|
@ -1933,21 +1915,9 @@ _PyXI_InitTypes(PyInterpreterState *interp)
|
||||||
void
|
void
|
||||||
_PyXI_FiniTypes(PyInterpreterState *interp)
|
_PyXI_FiniTypes(PyInterpreterState *interp)
|
||||||
{
|
{
|
||||||
_PyXI_state_t *state = _PyXI_GET_STATE(interp);
|
// We would finalize heap types here too but that leads to ref leaks.
|
||||||
if (state != NULL) {
|
// Instead, we finalize them in _PyXI_Fini().
|
||||||
// We would finalize heap types here too but that leads to ref leaks.
|
fini_static_exctypes(&_PyXI_GET_STATE(interp)->exceptions, interp);
|
||||||
// Instead, we finalize them in _PyXI_Fini().
|
|
||||||
fini_static_exctypes(&state->exceptions, interp);
|
|
||||||
|
|
||||||
xid_lookup_fini(&state->data_lookup);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_Py_IsMainInterpreter(interp)) {
|
|
||||||
_PyXI_global_state_t *global_state = _PyXI_GET_GLOBAL_STATE(interp);
|
|
||||||
if (global_state != NULL) {
|
|
||||||
xid_lookup_fini(&global_state->data_lookup);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -174,7 +174,6 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj)
|
||||||
PyTypeObject *cls = Py_TYPE(obj);
|
PyTypeObject *cls = Py_TYPE(obj);
|
||||||
|
|
||||||
dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls);
|
dlregistry_t *xidregistry = _get_xidregistry_for_type(ctx, cls);
|
||||||
assert(xidregistry->initialized);
|
|
||||||
_xidregistry_lock(xidregistry);
|
_xidregistry_lock(xidregistry);
|
||||||
|
|
||||||
dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls);
|
dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls);
|
||||||
|
@ -191,7 +190,6 @@ static int
|
||||||
_xidregistry_add_type(dlregistry_t *xidregistry,
|
_xidregistry_add_type(dlregistry_t *xidregistry,
|
||||||
PyTypeObject *cls, xidatafunc getdata)
|
PyTypeObject *cls, xidatafunc getdata)
|
||||||
{
|
{
|
||||||
assert(xidregistry->initialized);
|
|
||||||
dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t));
|
dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t));
|
||||||
if (newhead == NULL) {
|
if (newhead == NULL) {
|
||||||
return -1;
|
return -1;
|
||||||
|
|
|
@ -14,7 +14,6 @@
|
||||||
#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckRefcnt()
|
#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckRefcnt()
|
||||||
#include "pycore_initconfig.h" // _PyStatus_OK()
|
#include "pycore_initconfig.h" // _PyStatus_OK()
|
||||||
#include "pycore_long.h" // _PyLong_InitTypes()
|
#include "pycore_long.h" // _PyLong_InitTypes()
|
||||||
#include "pycore_memoryobject.h" // _PyMemoryView_InitTypes()
|
|
||||||
#include "pycore_object.h" // _PyDebug_PrintTotalRefs()
|
#include "pycore_object.h" // _PyDebug_PrintTotalRefs()
|
||||||
#include "pycore_obmalloc.h" // _PyMem_init_obmalloc()
|
#include "pycore_obmalloc.h" // _PyMem_init_obmalloc()
|
||||||
#include "pycore_optimizer.h" // _Py_Executors_InvalidateAll
|
#include "pycore_optimizer.h" // _Py_Executors_InvalidateAll
|
||||||
|
@ -755,11 +754,6 @@ pycore_init_types(PyInterpreterState *interp)
|
||||||
return status;
|
return status;
|
||||||
}
|
}
|
||||||
|
|
||||||
status = _PyMemoryView_InitTypes(interp);
|
|
||||||
if (_PyStatus_EXCEPTION(status)) {
|
|
||||||
return status;
|
|
||||||
}
|
|
||||||
|
|
||||||
return _PyStatus_OK();
|
return _PyStatus_OK();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1851,7 +1845,6 @@ finalize_interp_types(PyInterpreterState *interp)
|
||||||
_PyTypes_FiniExtTypes(interp);
|
_PyTypes_FiniExtTypes(interp);
|
||||||
_PyUnicode_FiniTypes(interp);
|
_PyUnicode_FiniTypes(interp);
|
||||||
_PySys_FiniTypes(interp);
|
_PySys_FiniTypes(interp);
|
||||||
_PyMemoryView_FiniTypes(interp);
|
|
||||||
_PyXI_FiniTypes(interp);
|
_PyXI_FiniTypes(interp);
|
||||||
_PyExc_Fini(interp);
|
_PyExc_Fini(interp);
|
||||||
_PyFloat_FiniType(interp);
|
_PyFloat_FiniType(interp);
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue