mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-10-09 08:31:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 13

gh-76785: Fixes for test.support.interpreters (gh-112982)

Browse source 
This involves a number of changes for PEP 734.
This commit is contained in:
Eric Snow 2023-12-12 08:24:31 -07:00 • committed by GitHub
parent f26bfe4b25
commit 86a77f4e1a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 2506 additions and 1507 deletions
Download patch file Download diff file Expand all files Collapse all files

34
Modules/_testcapimodule.c
View file

@ -13,6 +13,7 @@
#include "_testcapi/parts.h"
#include "frameobject.h" // PyFrame_New()
#include "interpreteridobject.h" // PyInterpreterID_Type
#include "marshal.h" // PyMarshal_WriteLongToFile()
#include <float.h> // FLT_MAX
@ -1451,6 +1452,36 @@ run_in_subinterp(PyObject *self, PyObject *args)
return PyLong_FromLong(r);
}
static PyObject *
get_interpreterid_type(PyObject *self, PyObject *Py_UNUSED(ignored))
{
return Py_NewRef(&PyInterpreterID_Type);
}
static PyObject *
link_interpreter_refcount(PyObject *self, PyObject *idobj)
{
PyInterpreterState *interp = PyInterpreterID_LookUp(idobj);
if (interp == NULL) {
assert(PyErr_Occurred());
return NULL;
}
_PyInterpreterState_RequireIDRef(interp, 1);
Py_RETURN_NONE;
}
static PyObject *
unlink_interpreter_refcount(PyObject *self, PyObject *idobj)
{
PyInterpreterState *interp = PyInterpreterID_LookUp(idobj);
if (interp == NULL) {
assert(PyErr_Occurred());
return NULL;
}
_PyInterpreterState_RequireIDRef(interp, 0);
Py_RETURN_NONE;
}
static PyMethodDef ml;
static PyObject *
@ -3237,6 +3268,9 @@ static PyMethodDef TestMethods[] = {
{"crash_no_current_thread", crash_no_current_thread, METH_NOARGS},
{"test_current_tstate_matches", test_current_tstate_matches, METH_NOARGS},
{"run_in_subinterp", run_in_subinterp, METH_VARARGS},
{"get_interpreterid_type", get_interpreterid_type, METH_NOARGS},
{"link_interpreter_refcount", link_interpreter_refcount, METH_O},
{"unlink_interpreter_refcount", unlink_interpreter_refcount, METH_O},
{"create_cfunction", create_cfunction, METH_NOARGS},
{"call_in_temporary_c_thread", call_in_temporary_c_thread, METH_VARARGS,
PyDoc_STR("set_error_class(error_class) -> None")},

12
Modules/_testinternalcapi.c
View file

@ -1475,6 +1475,17 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
}
static PyObject *
get_interpreter_refcount(PyObject *self, PyObject *idobj)
{
PyInterpreterState *interp = PyInterpreterID_LookUp(idobj);
if (interp == NULL) {
return NULL;
}
return PyLong_FromLongLong(interp->id_refcount);
}
static void
_xid_capsule_destructor(PyObject *capsule)
{
@ -1693,6 +1704,7 @@ static PyMethodDef module_functions[] = {
{"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
{"get_interpreter_refcount", get_interpreter_refcount, METH_O},
{"compile_perf_trampoline_entry", compile_perf_trampoline_entry, METH_VARARGS},
{"perf_trampoline_set_persist_after_fork", perf_trampoline_set_persist_after_fork, METH_VARARGS},
{"get_crossinterp_data", get_crossinterp_data, METH_VARARGS},

157
Modules/_xxinterpchannelsmodule.c
View file

@ -8,7 +8,6 @@
#include "Python.h"
#include "interpreteridobject.h"
#include "pycore_crossinterp.h" // struct _xid
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
#include "pycore_interp.h" // _PyInterpreterState_LookUpID()
#ifdef MS_WINDOWS
@ -263,136 +262,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout)
}
/* Cross-interpreter Buffer Views *******************************************/
// XXX Release when the original interpreter is destroyed.
typedef struct {
PyObject_HEAD
Py_buffer *view;
int64_t interpid;
} XIBufferViewObject;
static PyObject *
xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data)
{
assert(data->data != NULL);
assert(data->obj == NULL);
assert(data->interpid >= 0);
XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject));
if (self == NULL) {
return NULL;
}
PyObject_Init((PyObject *)self, cls);
self->view = (Py_buffer *)data->data;
self->interpid = data->interpid;
return (PyObject *)self;
}
static void
xibufferview_dealloc(XIBufferViewObject *self)
{
PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid);
/* If the interpreter is no longer alive then we have problems,
since other objects may be using the buffer still. */
assert(interp != NULL);
if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) {
// XXX Emit a warning?
PyErr_Clear();
}
PyTypeObject *tp = Py_TYPE(self);
tp->tp_free(self);
/* "Instances of heap-allocated types hold a reference to their type."
* See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
* See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
*/
// XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
// like we do for _abc._abc_data?
Py_DECREF(tp);
}
static int
xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags)
{
/* Only PyMemoryView_FromObject() should ever call this,
via _memoryview_from_xid() below. */
*view = *self->view;
view->obj = (PyObject *)self;
// XXX Should we leave it alone?
view->internal = NULL;
return 0;
}
static PyType_Slot XIBufferViewType_slots[] = {
{Py_tp_dealloc, (destructor)xibufferview_dealloc},
{Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf},
// We don't bother with Py_bf_releasebuffer since we don't need it.
{0, NULL},
};
static PyType_Spec XIBufferViewType_spec = {
.name = MODULE_NAME ".CrossInterpreterBufferView",
.basicsize = sizeof(XIBufferViewObject),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
.slots = XIBufferViewType_slots,
};
/* extra XID types **********************************************************/
static PyTypeObject * _get_current_xibufferview_type(void);
static PyObject *
_memoryview_from_xid(_PyCrossInterpreterData *data)
{
PyTypeObject *cls = _get_current_xibufferview_type();
if (cls == NULL) {
return NULL;
}
PyObject *obj = xibufferview_from_xid(cls, data);
if (obj == NULL) {
return NULL;
}
return PyMemoryView_FromObject(obj);
}
static int
_memoryview_shared(PyThreadState *tstate, PyObject *obj,
_PyCrossInterpreterData *data)
{
Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer));
if (view == NULL) {
return -1;
}
if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) {
PyMem_RawFree(view);
return -1;
}
_PyCrossInterpreterData_Init(data, tstate->interp, view, NULL,
_memoryview_from_xid);
return 0;
}
static int
register_builtin_xid_types(struct xid_class_registry *classes)
{
PyTypeObject *cls;
crossinterpdatafunc func;
// builtin memoryview
cls = &PyMemoryView_Type;
func = _memoryview_shared;
if (register_xid_class(cls, func, classes)) {
return -1;
}
return 0;
}
/* module state *************************************************************/
typedef struct {
@ -405,7 +274,6 @@ typedef struct {
/* heap types */
PyTypeObject *ChannelInfoType;
PyTypeObject *ChannelIDType;
PyTypeObject *XIBufferViewType;
/* exceptions */
PyObject *ChannelError;
@ -449,7 +317,6 @@ traverse_module_state(module_state *state, visitproc visit, void *arg)
/* heap types */
Py_VISIT(state->ChannelInfoType);
Py_VISIT(state->ChannelIDType);
Py_VISIT(state->XIBufferViewType);
/* exceptions */
Py_VISIT(state->ChannelError);
@ -474,7 +341,6 @@ clear_module_state(module_state *state)
(void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType);
}
Py_CLEAR(state->ChannelIDType);
Py_CLEAR(state->XIBufferViewType);
/* exceptions */
Py_CLEAR(state->ChannelError);
@ -487,17 +353,6 @@ clear_module_state(module_state *state)
}
static PyTypeObject *
_get_current_xibufferview_type(void)
{
module_state *state = _get_current_module_state();
if (state == NULL) {
return NULL;
}
return state->XIBufferViewType;
}
/* channel-specific code ****************************************************/
#define CHANNEL_SEND 1
@ -3463,18 +3318,6 @@ module_exec(PyObject *mod)
goto error;
}
// XIBufferView
state->XIBufferViewType = add_new_type(mod, &XIBufferViewType_spec, NULL,
xid_classes);
if (state->XIBufferViewType == NULL) {
goto error;
}
// Register external types.
if (register_builtin_xid_types(xid_classes) < 0) {
goto error;
}
/* Make sure chnnels drop objects owned by this interpreter. */
PyInterpreterState *interp = _get_current_interp();
PyUnstable_AtExit(interp, clear_interpreter, (void *)interp);

367
Modules/_xxsubinterpretersmodule.c
View file

@ -6,11 +6,14 @@
#endif
#include "Python.h"
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_crossinterp.h" // struct _xid
#include "pycore_pyerrors.h" // _Py_excinfo
#include "pycore_interp.h" // _PyInterpreterState_IDIncref()
#include "pycore_initconfig.h" // _PyErr_SetFromPyStatus()
#include "pycore_long.h" // _PyLong_IsNegative()
#include "pycore_modsupport.h" // _PyArg_BadArgument()
#include "pycore_pyerrors.h" // _PyErr_ChainExceptions1()
#include "pycore_pybuffer.h" // _PyBuffer_ReleaseInInterpreterAndRawFree()
#include "pycore_pyerrors.h" // _Py_excinfo
#include "pycore_pystate.h" // _PyInterpreterState_SetRunningMain()
#include "interpreteridobject.h"
@ -28,11 +31,260 @@ _get_current_interp(void)
return PyInterpreterState_Get();
}
static int64_t
pylong_to_interpid(PyObject *idobj)
{
assert(PyLong_CheckExact(idobj));
if (_PyLong_IsNegative((PyLongObject *)idobj)) {
PyErr_Format(PyExc_ValueError,
"interpreter ID must be a non-negative int, got %R",
idobj);
return -1;
}
int overflow;
long long id = PyLong_AsLongLongAndOverflow(idobj, &overflow);
if (id == -1) {
if (!overflow) {
assert(PyErr_Occurred());
return -1;
}
assert(!PyErr_Occurred());
// For now, we don't worry about if LLONG_MAX < INT64_MAX.
goto bad_id;
}
#if LLONG_MAX > INT64_MAX
if (id > INT64_MAX) {
goto bad_id;
}
#endif
return (int64_t)id;
bad_id:
PyErr_Format(PyExc_RuntimeError,
"unrecognized interpreter ID %O", idobj);
return -1;
}
static int64_t
convert_interpid_obj(PyObject *arg)
{
int64_t id = -1;
if (_PyIndex_Check(arg)) {
PyObject *idobj = PyNumber_Long(arg);
if (idobj == NULL) {
return -1;
}
id = pylong_to_interpid(idobj);
Py_DECREF(idobj);
if (id < 0) {
return -1;
}
}
else {
PyErr_Format(PyExc_TypeError,
"interpreter ID must be an int, got %.100s",
Py_TYPE(arg)->tp_name);
return -1;
}
return id;
}
static PyInterpreterState *
look_up_interp(PyObject *arg)
{
int64_t id = convert_interpid_obj(arg);
if (id < 0) {
return NULL;
}
return _PyInterpreterState_LookUpID(id);
}
static PyObject *
interpid_to_pylong(int64_t id)
{
assert(id < LLONG_MAX);
return PyLong_FromLongLong(id);
}
static PyObject *
get_interpid_obj(PyInterpreterState *interp)
{
if (_PyInterpreterState_IDInitref(interp) != 0) {
return NULL;
};
int64_t id = PyInterpreterState_GetID(interp);
if (id < 0) {
return NULL;
}
return interpid_to_pylong(id);
}
static PyObject *
_get_current_module(void)
{
PyObject *name = PyUnicode_FromString(MODULE_NAME);
if (name == NULL) {
return NULL;
}
PyObject *mod = PyImport_GetModule(name);
Py_DECREF(name);
if (mod == NULL) {
return NULL;
}
assert(mod != Py_None);
return mod;
}
/* Cross-interpreter Buffer Views *******************************************/
// XXX Release when the original interpreter is destroyed.
typedef struct {
PyObject_HEAD
Py_buffer *view;
int64_t interpid;
} XIBufferViewObject;
static PyObject *
xibufferview_from_xid(PyTypeObject *cls, _PyCrossInterpreterData *data)
{
assert(data->data != NULL);
assert(data->obj == NULL);
assert(data->interpid >= 0);
XIBufferViewObject *self = PyObject_Malloc(sizeof(XIBufferViewObject));
if (self == NULL) {
return NULL;
}
PyObject_Init((PyObject *)self, cls);
self->view = (Py_buffer *)data->data;
self->interpid = data->interpid;
return (PyObject *)self;
}
static void
xibufferview_dealloc(XIBufferViewObject *self)
{
PyInterpreterState *interp = _PyInterpreterState_LookUpID(self->interpid);
/* If the interpreter is no longer alive then we have problems,
since other objects may be using the buffer still. */
assert(interp != NULL);
if (_PyBuffer_ReleaseInInterpreterAndRawFree(interp, self->view) < 0) {
// XXX Emit a warning?
PyErr_Clear();
}
PyTypeObject *tp = Py_TYPE(self);
tp->tp_free(self);
/* "Instances of heap-allocated types hold a reference to their type."
* See: https://docs.python.org/3.11/howto/isolating-extensions.html#garbage-collection-protocol
* See: https://docs.python.org/3.11/c-api/typeobj.html#c.PyTypeObject.tp_traverse
*/
// XXX Why don't we implement Py_TPFLAGS_HAVE_GC, e.g. Py_tp_traverse,
// like we do for _abc._abc_data?
Py_DECREF(tp);
}
static int
xibufferview_getbuf(XIBufferViewObject *self, Py_buffer *view, int flags)
{
/* Only PyMemoryView_FromObject() should ever call this,
via _memoryview_from_xid() below. */
*view = *self->view;
view->obj = (PyObject *)self;
// XXX Should we leave it alone?
view->internal = NULL;
return 0;
}
static PyType_Slot XIBufferViewType_slots[] = {
{Py_tp_dealloc, (destructor)xibufferview_dealloc},
{Py_bf_getbuffer, (getbufferproc)xibufferview_getbuf},
// We don't bother with Py_bf_releasebuffer since we don't need it.
{0, NULL},
};
static PyType_Spec XIBufferViewType_spec = {
.name = MODULE_NAME ".CrossInterpreterBufferView",
.basicsize = sizeof(XIBufferViewObject),
.flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_IMMUTABLETYPE),
.slots = XIBufferViewType_slots,
};
static PyTypeObject * _get_current_xibufferview_type(void);
static PyObject *
_memoryview_from_xid(_PyCrossInterpreterData *data)
{
PyTypeObject *cls = _get_current_xibufferview_type();
if (cls == NULL) {
return NULL;
}
PyObject *obj = xibufferview_from_xid(cls, data);
if (obj == NULL) {
return NULL;
}
return PyMemoryView_FromObject(obj);
}
static int
_memoryview_shared(PyThreadState *tstate, PyObject *obj,
_PyCrossInterpreterData *data)
{
Py_buffer *view = PyMem_RawMalloc(sizeof(Py_buffer));
if (view == NULL) {
return -1;
}
if (PyObject_GetBuffer(obj, view, PyBUF_FULL_RO) < 0) {
PyMem_RawFree(view);
return -1;
}
_PyCrossInterpreterData_Init(data, tstate->interp, view, NULL,
_memoryview_from_xid);
return 0;
}
static int
register_memoryview_xid(PyObject *mod, PyTypeObject **p_state)
{
// XIBufferView
assert(*p_state == NULL);
PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec(
mod, &XIBufferViewType_spec, NULL);
if (cls == NULL) {
return -1;
}
if (PyModule_AddType(mod, cls) < 0) {
Py_DECREF(cls);
return -1;
}
*p_state = cls;
// Register XID for the builtin memoryview type.
if (_PyCrossInterpreterData_RegisterClass(
&PyMemoryView_Type, _memoryview_shared) < 0) {
return -1;
}
// We don't ever bother un-registering memoryview.
return 0;
}
/* module state *************************************************************/
typedef struct {
int _notused;
/* heap types */
PyTypeObject *XIBufferViewType;
} module_state;
static inline module_state *
@ -44,19 +296,51 @@ get_module_state(PyObject *mod)
return state;
}
static module_state *
_get_current_module_state(void)
{
PyObject *mod = _get_current_module();
if (mod == NULL) {
// XXX import it?
PyErr_SetString(PyExc_RuntimeError,
MODULE_NAME " module not imported yet");
return NULL;
}
module_state *state = get_module_state(mod);
Py_DECREF(mod);
return state;
}
static int
traverse_module_state(module_state *state, visitproc visit, void *arg)
{
/* heap types */
Py_VISIT(state->XIBufferViewType);
return 0;
}
static int
clear_module_state(module_state *state)
{
/* heap types */
Py_CLEAR(state->XIBufferViewType);
return 0;
}
static PyTypeObject *
_get_current_xibufferview_type(void)
{
module_state *state = _get_current_module_state();
if (state == NULL) {
return NULL;
}
return state->XIBufferViewType;
}
/* Python code **************************************************************/
static const char *
@ -254,7 +538,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
PyObject *idobj = PyInterpreterState_GetIDObject(interp);
PyObject *idobj = get_interpid_obj(interp);
if (idobj == NULL) {
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
@ -273,7 +557,9 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
PyDoc_STRVAR(create_doc,
"create() -> ID\n\
\n\
Create a new interpreter and return a unique generated ID.");
Create a new interpreter and return a unique generated ID.\n\
\n\
The caller is responsible for destroying the interpreter before exiting.");
static PyObject *
@ -288,7 +574,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
}
// Look up the interpreter.
PyInterpreterState *interp = PyInterpreterID_LookUp(id);
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
@ -345,7 +631,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
interp = PyInterpreterState_Head();
while (interp != NULL) {
id = PyInterpreterState_GetIDObject(interp);
id = get_interpid_obj(interp);
if (id == NULL) {
Py_DECREF(ids);
return NULL;
@ -377,7 +663,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
if (interp == NULL) {
return NULL;
}
return PyInterpreterState_GetIDObject(interp);
return get_interpid_obj(interp);
}
PyDoc_STRVAR(get_current_doc,
@ -391,7 +677,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
{
// Currently, 0 is always the main interpreter.
int64_t id = 0;
return PyInterpreterID_New(id);
return PyLong_FromLongLong(id);
}
PyDoc_STRVAR(get_main_doc,
@ -481,7 +767,7 @@ _interp_exec(PyObject *self,
PyObject **p_excinfo)
{
// Look up the interpreter.
PyInterpreterState *interp = PyInterpreterID_LookUp(id_arg);
PyInterpreterState *interp = look_up_interp(id_arg);
if (interp == NULL) {
return -1;
}
@ -667,7 +953,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyInterpreterState *interp = PyInterpreterID_LookUp(id);
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
@ -683,6 +969,49 @@ PyDoc_STRVAR(is_running_doc,
Return whether or not the identified interpreter is running.");
static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:_incref", kwlist, &id)) {
return NULL;
}
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
if (_PyInterpreterState_IDInitref(interp) < 0) {
return NULL;
}
_PyInterpreterState_IDIncref(interp);
Py_RETURN_NONE;
}
static PyObject *
interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:_incref", kwlist, &id)) {
return NULL;
}
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
_PyInterpreterState_IDDecref(interp);
Py_RETURN_NONE;
}
static PyMethodDef module_functions[] = {
{"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc},
@ -707,6 +1036,11 @@ static PyMethodDef module_functions[] = {
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"_incref", _PyCFunction_CAST(interp_incref),
METH_VARARGS | METH_KEYWORDS, NULL},
{"_decref", _PyCFunction_CAST(interp_decref),
METH_VARARGS | METH_KEYWORDS, NULL},
{NULL, NULL} /* sentinel */
};
@ -720,8 +1054,17 @@ The 'interpreters' module provides a more convenient interface.");
static int
module_exec(PyObject *mod)
{
// PyInterpreterID
if (PyModule_AddType(mod, &PyInterpreterID_Type) < 0) {
module_state *state = get_module_state(mod);
// exceptions
if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) {
goto error;
}
if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterNotFoundError) < 0) {
goto error;
}
if (register_memoryview_xid(mod, &state->XIBufferViewType) < 0) {
goto error;
}