gh-76785: Add More Tests to test_interpreters.test_api (gh-117662)

In addition to the increase test coverage, this is a precursor to sorting out how we handle interpreters created directly via the C-API.
This commit is contained in:
Eric Snow 2024-04-10 18:37:01 -06:00 committed by GitHub
parent 0cc71bde00
commit 993c3cca16
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 2015 additions and 421 deletions

View file

@ -19,20 +19,3 @@ clear_xid_class(PyTypeObject *cls)
return _PyCrossInterpreterData_UnregisterClass(cls);
}
#endif
#ifdef RETURNS_INTERPID_OBJECT
static PyObject *
get_interpid_obj(PyInterpreterState *interp)
{
if (_PyInterpreterState_IDInitref(interp) != 0) {
return NULL;
};
int64_t id = PyInterpreterState_GetID(interp);
if (id < 0) {
return NULL;
}
assert(id < LLONG_MAX);
return PyLong_FromLongLong(id);
}
#endif

View file

@ -1369,56 +1369,284 @@ dict_getitem_knownhash(PyObject *self, PyObject *args)
}
/* To run some code in a sub-interpreter. */
static int
_init_interp_config_from_object(PyInterpreterConfig *config, PyObject *obj)
{
if (obj == NULL) {
*config = (PyInterpreterConfig)_PyInterpreterConfig_INIT;
return 0;
}
PyObject *dict = PyObject_GetAttrString(obj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", obj);
return -1;
}
int res = _PyInterpreterConfig_InitFromDict(config, dict);
Py_DECREF(dict);
if (res < 0) {
return -1;
}
return 0;
}
static PyInterpreterState *
_new_interpreter(PyInterpreterConfig *config, long whence)
{
if (whence == _PyInterpreterState_WHENCE_XI) {
return _PyXI_NewInterpreter(config, NULL, NULL);
}
PyObject *exc = NULL;
PyInterpreterState *interp = NULL;
if (whence == _PyInterpreterState_WHENCE_UNKNOWN) {
assert(config == NULL);
interp = PyInterpreterState_New();
}
else if (whence == _PyInterpreterState_WHENCE_CAPI
|| whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
{
PyThreadState *tstate = NULL;
PyThreadState *save_tstate = PyThreadState_Swap(NULL);
if (whence == _PyInterpreterState_WHENCE_LEGACY_CAPI) {
assert(config == NULL);
tstate = Py_NewInterpreter();
PyThreadState_Swap(save_tstate);
}
else {
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
assert(tstate == NULL);
_PyErr_SetFromPyStatus(status);
exc = PyErr_GetRaisedException();
}
}
if (tstate != NULL) {
interp = PyThreadState_GetInterpreter(tstate);
// Throw away the initial tstate.
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
}
}
else {
PyErr_Format(PyExc_ValueError,
"unsupported whence %ld", whence);
return NULL;
}
if (interp == NULL) {
PyErr_SetString(PyExc_InterpreterError,
"sub-interpreter creation failed");
if (exc != NULL) {
_PyErr_ChainExceptions1(exc);
}
}
return interp;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.create()
static PyObject *
create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"config", "whence", NULL};
PyObject *configobj = NULL;
long whence = _PyInterpreterState_WHENCE_XI;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"|O$l:create_interpreter", kwlist,
&configobj, &whence))
{
return NULL;
}
if (configobj == Py_None) {
configobj = NULL;
}
// Resolve the config.
PyInterpreterConfig *config = NULL;
PyInterpreterConfig _config;
if (whence == _PyInterpreterState_WHENCE_UNKNOWN
|| whence == _PyInterpreterState_WHENCE_LEGACY_CAPI)
{
if (configobj != NULL) {
PyErr_SetString(PyExc_ValueError, "got unexpected config");
return NULL;
}
}
else {
config = &_config;
if (_init_interp_config_from_object(config, configobj) < 0) {
return NULL;
}
}
// Create the interpreter.
PyInterpreterState *interp = _new_interpreter(config, whence);
if (interp == NULL) {
return NULL;
}
// Return the ID.
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
_PyXI_EndInterpreter(interp, NULL, NULL);
return NULL;
}
return idobj;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.destroy()
static PyObject *
destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"id", NULL};
PyObject *idobj = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"O:destroy_interpreter", kwlist,
&idobj))
{
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
_PyXI_EndInterpreter(interp, NULL, NULL);
Py_RETURN_NONE;
}
// This exists mostly for testing the _interpreters module, as an
// alternative to _interpreters.destroy()
static PyObject *
exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs)
{
static char *kwlist[] = {"id", "code", "main", NULL};
PyObject *idobj;
const char *code;
int runningmain = 0;
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"Os|$p:exec_interpreter", kwlist,
&idobj, &code, &runningmain))
{
return NULL;
}
PyInterpreterState *interp = _PyInterpreterState_LookUpIDObject(idobj);
if (interp == NULL) {
return NULL;
}
PyObject *res = NULL;
PyThreadState *tstate = PyThreadState_New(interp);
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_EXEC);
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
if (runningmain) {
if (_PyInterpreterState_SetRunningMain(interp) < 0) {
goto finally;
}
}
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0};
int r = PyRun_SimpleStringFlags(code, &cflags);
if (PyErr_Occurred()) {
PyErr_PrintEx(0);
}
if (runningmain) {
_PyInterpreterState_SetNotRunningMain(interp);
}
res = PyLong_FromLong(r);
finally:
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
return res;
}
/* To run some code in a sub-interpreter.
Generally you can use test.support.interpreters,
but we keep this helper as a distinct implementation.
That's especially important for testing test.support.interpreters.
*/
static PyObject *
run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs)
{
const char *code;
PyObject *configobj;
static char *kwlist[] = {"code", "config", NULL};
int xi = 0;
static char *kwlist[] = {"code", "config", "xi", NULL};
if (!PyArg_ParseTupleAndKeywords(args, kwargs,
"sO:run_in_subinterp_with_config", kwlist,
&code, &configobj))
"sO|$p:run_in_subinterp_with_config", kwlist,
&code, &configobj, &xi))
{
return NULL;
}
PyInterpreterConfig config;
PyObject *dict = PyObject_GetAttrString(configobj, "__dict__");
if (dict == NULL) {
PyErr_Format(PyExc_TypeError, "bad config %R", configobj);
return NULL;
}
int res = _PyInterpreterConfig_InitFromDict(&config, dict);
Py_DECREF(dict);
if (res < 0) {
if (_init_interp_config_from_object(&config, configobj) < 0) {
return NULL;
}
PyThreadState *mainstate = PyThreadState_Get();
PyThreadState_Swap(NULL);
PyThreadState *substate;
PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
PyThreadState_Swap(mainstate);
_PyErr_SetFromPyStatus(status);
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_RuntimeError, "sub-interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
assert(substate != NULL);
/* only initialise 'cflags.cf_flags' to test backwards compatibility */
PyCompilerFlags cflags = {0};
int r = PyRun_SimpleStringFlags(code, &cflags);
Py_EndInterpreter(substate);
PyThreadState_Swap(mainstate);
int r;
if (xi) {
PyThreadState *save_tstate;
PyThreadState *tstate;
/* Create an interpreter, staying switched to it. */
PyInterpreterState *interp = \
_PyXI_NewInterpreter(&config, &tstate, &save_tstate);
if (interp == NULL) {
return NULL;
}
/* Exec the code in the new interpreter. */
r = PyRun_SimpleStringFlags(code, &cflags);
/* clean up post-exec. */
_PyXI_EndInterpreter(interp, tstate, &save_tstate);
}
else {
PyThreadState *substate;
PyThreadState *mainstate = PyThreadState_Swap(NULL);
/* Create an interpreter, staying switched to it. */
PyStatus status = Py_NewInterpreterFromConfig(&substate, &config);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
PyThreadState_Swap(mainstate);
_PyErr_SetFromPyStatus(status);
PyObject *exc = PyErr_GetRaisedException();
PyErr_SetString(PyExc_InterpreterError,
"sub-interpreter creation failed");
_PyErr_ChainExceptions1(exc);
return NULL;
}
/* Exec the code in the new interpreter. */
r = PyRun_SimpleStringFlags(code, &cflags);
/* clean up post-exec. */
Py_EndInterpreter(substate);
PyThreadState_Swap(mainstate);
}
return PyLong_FromLong(r);
}
@ -1434,6 +1662,13 @@ normalize_interp_id(PyObject *self, PyObject *idobj)
return PyLong_FromLongLong(interpid);
}
static PyObject *
next_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
{
int64_t interpid = _PyRuntime.interpreters.next_id;
return PyLong_FromLongLong(interpid);
}
static PyObject *
unused_interpreter_id(PyObject *self, PyObject *Py_UNUSED(ignored))
{
@ -1751,10 +1986,17 @@ static PyMethodDef module_functions[] = {
{"get_object_dict_values", get_object_dict_values, METH_O},
{"hamt", new_hamt, METH_NOARGS},
{"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS},
{"create_interpreter", _PyCFunction_CAST(create_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"destroy_interpreter", _PyCFunction_CAST(destroy_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"exec_interpreter", _PyCFunction_CAST(exec_interpreter),
METH_VARARGS | METH_KEYWORDS},
{"run_in_subinterp_with_config",
_PyCFunction_CAST(run_in_subinterp_with_config),
METH_VARARGS | METH_KEYWORDS},
{"normalize_interp_id", normalize_interp_id, METH_O},
{"next_interpreter_id", next_interpreter_id, METH_NOARGS},
{"unused_interpreter_id", unused_interpreter_id, METH_NOARGS},
{"interpreter_exists", interpreter_exists, METH_O},
{"get_interpreter_refcount", get_interpreter_refcount, METH_O},

View file

@ -8,6 +8,7 @@
#include "Python.h"
#include "pycore_crossinterp.h" // struct _xid
#include "pycore_interp.h" // _PyInterpreterState_LookUpID()
#include "pycore_pystate.h" // _PyInterpreterState_GetIDObject()
#ifdef MS_WINDOWS
#define WIN32_LEAN_AND_MEAN
@ -17,9 +18,7 @@
#endif
#define REGISTERS_HEAP_TYPES
#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h"
#undef RETURNS_INTERPID_OBJECT
#undef REGISTERS_HEAP_TYPES
@ -2909,7 +2908,7 @@ channelsmod_list_interpreters(PyObject *self, PyObject *args, PyObject *kwds)
goto except;
}
if (res) {
interpid_obj = get_interpid_obj(interp);
interpid_obj = _PyInterpreterState_GetIDObject(interp);
if (interpid_obj == NULL) {
goto except;
}

View file

@ -20,9 +20,7 @@
#include "marshal.h" // PyMarshal_ReadObjectFromString()
#define RETURNS_INTERPID_OBJECT
#include "_interpreters_common.h"
#undef RETURNS_INTERPID_OBJECT
#define MODULE_NAME _xxsubinterpreters
@ -425,59 +423,6 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config)
}
static PyInterpreterState *
new_interpreter(PyInterpreterConfig *config, PyObject **p_idobj, PyThreadState **p_tstate)
{
PyThreadState *save_tstate = PyThreadState_Get();
assert(save_tstate != NULL);
PyThreadState *tstate = NULL;
// XXX Possible GILState issues?
PyStatus status = Py_NewInterpreterFromConfig(&tstate, config);
PyThreadState_Swap(save_tstate);
if (PyStatus_Exception(status)) {
/* Since no new thread state was created, there is no exception to
propagate; raise a fresh one after swapping in the old thread
state. */
_PyErr_SetFromPyStatus(status);
return NULL;
}
assert(tstate != NULL);
PyInterpreterState *interp = PyThreadState_GetInterpreter(tstate);
if (_PyInterpreterState_IDInitref(interp) < 0) {
goto error;
}
if (p_idobj != NULL) {
// We create the object using the original interpreter.
PyObject *idobj = get_interpid_obj(interp);
if (idobj == NULL) {
goto error;
}
*p_idobj = idobj;
}
if (p_tstate != NULL) {
*p_tstate = tstate;
}
else {
PyThreadState_Swap(tstate);
PyThreadState_Clear(tstate);
PyThreadState_Swap(save_tstate);
PyThreadState_Delete(tstate);
}
return interp;
error:
// XXX Possible GILState issues?
save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
return NULL;
}
static int
_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags)
{
@ -546,6 +491,19 @@ _run_in_interpreter(PyInterpreterState *interp,
/* module level code ********************************************************/
static PyObject *
get_summary(PyInterpreterState *interp)
{
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
return NULL;
}
PyObject *res = PyTuple_Pack(1, idobj);
Py_DECREF(idobj);
return res;
}
static PyObject *
interp_new_config(PyObject *self, PyObject *args, PyObject *kwds)
{
@ -606,8 +564,7 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyObject *idobj = NULL;
PyInterpreterState *interp = new_interpreter(&config, &idobj, NULL);
PyInterpreterState *interp = _PyXI_NewInterpreter(&config, NULL, NULL);
if (interp == NULL) {
// XXX Move the chained exception to interpreters.create()?
PyObject *exc = PyErr_GetRaisedException();
@ -617,6 +574,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds)
return NULL;
}
PyObject *idobj = _PyInterpreterState_GetIDObject(interp);
if (idobj == NULL) {
_PyXI_EndInterpreter(interp, NULL, NULL);
return NULL;
}
if (reqrefs) {
// Decref to 0 will destroy the interpreter.
_PyInterpreterState_RequireIDRef(interp, 1);
@ -678,12 +641,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
}
// Destroy the interpreter.
PyThreadState *tstate = PyThreadState_New(interp);
_PyThreadState_SetWhence(tstate, _PyThreadState_WHENCE_INTERP);
// XXX Possible GILState issues?
PyThreadState *save_tstate = PyThreadState_Swap(tstate);
Py_EndInterpreter(tstate);
PyThreadState_Swap(save_tstate);
_PyXI_EndInterpreter(interp, NULL, NULL);
Py_RETURN_NONE;
}
@ -700,7 +658,7 @@ So does an unrecognized ID.");
static PyObject *
interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
{
PyObject *ids, *id;
PyObject *ids;
PyInterpreterState *interp;
ids = PyList_New(0);
@ -710,14 +668,14 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
interp = PyInterpreterState_Head();
while (interp != NULL) {
id = get_interpid_obj(interp);
if (id == NULL) {
PyObject *item = get_summary(interp);
if (item == NULL) {
Py_DECREF(ids);
return NULL;
}
// insert at front of list
int res = PyList_Insert(ids, 0, id);
Py_DECREF(id);
int res = PyList_Insert(ids, 0, item);
Py_DECREF(item);
if (res < 0) {
Py_DECREF(ids);
return NULL;
@ -730,7 +688,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
}
PyDoc_STRVAR(list_all_doc,
"list_all() -> [ID]\n\
"list_all() -> [(ID,)]\n\
\n\
Return a list containing the ID of every existing interpreter.");
@ -742,11 +700,11 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
if (interp == NULL) {
return NULL;
}
return get_interpid_obj(interp);
return get_summary(interp);
}
PyDoc_STRVAR(get_current_doc,
"get_current() -> ID\n\
"get_current() -> (ID,)\n\
\n\
Return the ID of current interpreter.");
@ -754,13 +712,12 @@ Return the ID of current interpreter.");
static PyObject *
interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
{
// Currently, 0 is always the main interpreter.
int64_t id = 0;
return PyLong_FromLongLong(id);
PyInterpreterState *interp = _PyInterpreterState_Main();
return get_summary(interp);
}
PyDoc_STRVAR(get_main_doc,
"get_main() -> ID\n\
"get_main() -> (ID,)\n\
\n\
Return the ID of main interpreter.");
@ -1194,6 +1151,32 @@ PyDoc_STRVAR(get_config_doc,
Return a representation of the config used to initialize the interpreter.");
static PyObject *
interp_whence(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"id", NULL};
PyObject *id;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"O:whence", kwlist, &id))
{
return NULL;
}
PyInterpreterState *interp = look_up_interp(id);
if (interp == NULL) {
return NULL;
}
long whence = _PyInterpreterState_GetWhence(interp);
return PyLong_FromLong(whence);
}
PyDoc_STRVAR(whence_doc,
"whence(id) -> int\n\
\n\
Return an identifier for where the interpreter was created.");
static PyObject *
interp_incref(PyObject *self, PyObject *args, PyObject *kwds)
{
@ -1242,9 +1225,78 @@ interp_decref(PyObject *self, PyObject *args, PyObject *kwds)
}
static PyObject *
capture_exception(PyObject *self, PyObject *args, PyObject *kwds)
{
static char *kwlist[] = {"exc", NULL};
PyObject *exc_arg = NULL;
if (!PyArg_ParseTupleAndKeywords(args, kwds,
"|O:capture_exception", kwlist,
&exc_arg))
{
return NULL;
}
PyObject *exc = exc_arg;
if (exc == NULL || exc == Py_None) {
exc = PyErr_GetRaisedException();
if (exc == NULL) {
Py_RETURN_NONE;
}
}
else if (!PyExceptionInstance_Check(exc)) {
PyErr_Format(PyExc_TypeError, "expected exception, got %R", exc);
return NULL;
}
PyObject *captured = NULL;
_PyXI_excinfo info = {0};
if (_PyXI_InitExcInfo(&info, exc) < 0) {
goto finally;
}
captured = _PyXI_ExcInfoAsObject(&info);
if (captured == NULL) {
goto finally;
}
PyObject *formatted = _PyXI_FormatExcInfo(&info);
if (formatted == NULL) {
Py_CLEAR(captured);
goto finally;
}
int res = PyObject_SetAttrString(captured, "formatted", formatted);
Py_DECREF(formatted);
if (res < 0) {
Py_CLEAR(captured);
goto finally;
}
finally:
_PyXI_ClearExcInfo(&info);
if (exc != exc_arg) {
if (PyErr_Occurred()) {
PyErr_SetRaisedException(exc);
}
else {
_PyErr_ChainExceptions1(exc);
}
}
return captured;
}
PyDoc_STRVAR(capture_exception_doc,
"capture_exception(exc=None) -> types.SimpleNamespace\n\
\n\
Return a snapshot of an exception. If \"exc\" is None\n\
then the current exception, if any, is used (but not cleared).\n\
\n\
The returned snapshot is the same as what _interpreters.exec() returns.");
static PyMethodDef module_functions[] = {
{"new_config", _PyCFunction_CAST(interp_new_config),
METH_VARARGS | METH_KEYWORDS, new_config_doc},
{"create", _PyCFunction_CAST(interp_create),
METH_VARARGS | METH_KEYWORDS, create_doc},
{"destroy", _PyCFunction_CAST(interp_destroy),
@ -1260,6 +1312,8 @@ static PyMethodDef module_functions[] = {
METH_VARARGS | METH_KEYWORDS, is_running_doc},
{"get_config", _PyCFunction_CAST(interp_get_config),
METH_VARARGS | METH_KEYWORDS, get_config_doc},
{"whence", _PyCFunction_CAST(interp_whence),
METH_VARARGS | METH_KEYWORDS, whence_doc},
{"exec", _PyCFunction_CAST(interp_exec),
METH_VARARGS | METH_KEYWORDS, exec_doc},
{"call", _PyCFunction_CAST(interp_call),
@ -1271,14 +1325,18 @@ static PyMethodDef module_functions[] = {
{"set___main___attrs", _PyCFunction_CAST(interp_set___main___attrs),
METH_VARARGS, set___main___attrs_doc},
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"incref", _PyCFunction_CAST(interp_incref),
METH_VARARGS | METH_KEYWORDS, NULL},
{"decref", _PyCFunction_CAST(interp_decref),
METH_VARARGS | METH_KEYWORDS, NULL},
{"is_shareable", _PyCFunction_CAST(object_is_shareable),
METH_VARARGS | METH_KEYWORDS, is_shareable_doc},
{"capture_exception", _PyCFunction_CAST(capture_exception),
METH_VARARGS | METH_KEYWORDS, capture_exception_doc},
{NULL, NULL} /* sentinel */
};
@ -1295,6 +1353,19 @@ module_exec(PyObject *mod)
PyInterpreterState *interp = PyInterpreterState_Get();
module_state *state = get_module_state(mod);
#define ADD_WHENCE(NAME) \
if (PyModule_AddIntConstant(mod, "WHENCE_" #NAME, \
_PyInterpreterState_WHENCE_##NAME) < 0) \
{ \
goto error; \
}
ADD_WHENCE(UNKNOWN)
ADD_WHENCE(RUNTIME)
ADD_WHENCE(LEGACY_CAPI)
ADD_WHENCE(CAPI)
ADD_WHENCE(XI)
#undef ADD_WHENCE
// exceptions
if (PyModule_AddType(mod, (PyTypeObject *)PyExc_InterpreterError) < 0) {
goto error;