bpo-36097: Use only public C-API in the_xxsubinterpreters module (adding as necessary). (#12003)

Include/cpython/interpreteridobject.h

@@ -0,0 +1,21 @@
+#ifndef Py_CPYTHON_INTERPRETERIDOBJECT_H
+#  error "this header file must not be included directly"
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Interpreter ID Object */
+
+PyAPI_DATA(PyTypeObject) _PyInterpreterID_Type;
+
+PyAPI_FUNC(PyObject *) _PyInterpreterID_New(int64_t);
+PyAPI_FUNC(PyObject *) _PyInterpreterState_GetIDObject(PyInterpreterState *);
+PyAPI_FUNC(PyInterpreterState *) _PyInterpreterID_LookUp(PyObject *);
+
+PyAPI_FUNC(int64_t) _Py_CoerceID(PyObject *);
+
+#ifdef __cplusplus
+}
+#endif

Include/cpython/pystate.h

@@ -30,9 +30,13 @@ typedef struct {
     (_PyMainInterpreterConfig){.install_signal_handlers = -1}
 /* Note: _PyMainInterpreterConfig_INIT sets other fields to 0/NULL */
 
+PyAPI_FUNC(int) _PyInterpreterState_RequiresIDRef(PyInterpreterState *);
+PyAPI_FUNC(void) _PyInterpreterState_RequireIDRef(PyInterpreterState *, int);
+
 PyAPI_FUNC(_PyCoreConfig *) _PyInterpreterState_GetCoreConfig(PyInterpreterState *);
 PyAPI_FUNC(_PyMainInterpreterConfig *) _PyInterpreterState_GetMainConfig(PyInterpreterState *);
 
+PyAPI_FUNC(PyObject *) _PyInterpreterState_GetMainModule(PyInterpreterState *);
 
 /* State unique per thread */
 
@@ -214,6 +218,65 @@ PyAPI_FUNC(PyThreadState *) PyThreadState_Next(PyThreadState *);
 
 typedef struct _frame *(*PyThreadFrameGetter)(PyThreadState *self_);
 
+/* cross-interpreter data */
+
+struct _xid;
+
+// _PyCrossInterpreterData is similar to Py_buffer as an effectively
+// opaque struct that holds data outside the object machinery.  This
+// is necessary to pass safely between interpreters in the same process.
+typedef struct _xid {
+    // data is the cross-interpreter-safe derivation of a Python object
+    // (see _PyObject_GetCrossInterpreterData).  It will be NULL if the
+    // new_object func (below) encodes the data.
+    void *data;
+    // obj is the Python object from which the data was derived.  This
+    // is non-NULL only if the data remains bound to the object in some
+    // way, such that the object must be "released" (via a decref) when
+    // the data is released.  In that case the code that sets the field,
+    // likely a registered "crossinterpdatafunc", is responsible for
+    // ensuring it owns the reference (i.e. incref).
+    PyObject *obj;
+    // interp is the ID of the owning interpreter of the original
+    // object.  It corresponds to the active interpreter when
+    // _PyObject_GetCrossInterpreterData() was called.  This should only
+    // be set by the cross-interpreter machinery.
+    //
+    // We use the ID rather than the PyInterpreterState to avoid issues
+    // with deleted interpreters.  Note that IDs are never re-used, so
+    // each one will always correspond to a specific interpreter
+    // (whether still alive or not).
+    int64_t interp;
+    // new_object is a function that returns a new object in the current
+    // interpreter given the data.  The resulting object (a new
+    // reference) will be equivalent to the original object.  This field
+    // is required.
+    PyObject *(*new_object)(struct _xid *);
+    // free is called when the data is released.  If it is NULL then
+    // nothing will be done to free the data.  For some types this is
+    // okay (e.g. bytes) and for those types this field should be set
+    // to NULL.  However, for most the data was allocated just for
+    // cross-interpreter use, so it must be freed when
+    // _PyCrossInterpreterData_Release is called or the memory will
+    // leak.  In that case, at the very least this field should be set
+    // to PyMem_RawFree (the default if not explicitly set to NULL).
+    // The call will happen with the original interpreter activated.
+    void (*free)(void *);
+} _PyCrossInterpreterData;
+
+PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
+PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
+PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
+
+PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
+
+/* cross-interpreter data registry */
+
+typedef int (*crossinterpdatafunc)(PyObject *, struct _xid *);
+
+PyAPI_FUNC(int) _PyCrossInterpreterData_RegisterClass(PyTypeObject *, crossinterpdatafunc);
+PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_pystate.h

@@ -30,6 +30,7 @@ struct _is {
 
     int64_t id;
     int64_t id_refcount;
+    int requires_idref;
     PyThread_type_lock id_mutex;
 
     int finalizing;
@@ -92,66 +93,12 @@ PyAPI_FUNC(void) _PyInterpreterState_IDIncref(struct _is *);
 PyAPI_FUNC(void) _PyInterpreterState_IDDecref(struct _is *);
 
 
-/* cross-interpreter data */
-
-struct _xid;
-
-// _PyCrossInterpreterData is similar to Py_buffer as an effectively
-// opaque struct that holds data outside the object machinery.  This
-// is necessary to pass safely between interpreters in the same process.
-typedef struct _xid {
-    // data is the cross-interpreter-safe derivation of a Python object
-    // (see _PyObject_GetCrossInterpreterData).  It will be NULL if the
-    // new_object func (below) encodes the data.
-    void *data;
-    // obj is the Python object from which the data was derived.  This
-    // is non-NULL only if the data remains bound to the object in some
-    // way, such that the object must be "released" (via a decref) when
-    // the data is released.  In that case the code that sets the field,
-    // likely a registered "crossinterpdatafunc", is responsible for
-    // ensuring it owns the reference (i.e. incref).
-    PyObject *obj;
-    // interp is the ID of the owning interpreter of the original
-    // object.  It corresponds to the active interpreter when
-    // _PyObject_GetCrossInterpreterData() was called.  This should only
-    // be set by the cross-interpreter machinery.
-    //
-    // We use the ID rather than the PyInterpreterState to avoid issues
-    // with deleted interpreters.
-    int64_t interp;
-    // new_object is a function that returns a new object in the current
-    // interpreter given the data.  The resulting object (a new
-    // reference) will be equivalent to the original object.  This field
-    // is required.
-    PyObject *(*new_object)(struct _xid *);
-    // free is called when the data is released.  If it is NULL then
-    // nothing will be done to free the data.  For some types this is
-    // okay (e.g. bytes) and for those types this field should be set
-    // to NULL.  However, for most the data was allocated just for
-    // cross-interpreter use, so it must be freed when
-    // _PyCrossInterpreterData_Release is called or the memory will
-    // leak.  In that case, at the very least this field should be set
-    // to PyMem_RawFree (the default if not explicitly set to NULL).
-    // The call will happen with the original interpreter activated.
-    void (*free)(void *);
-} _PyCrossInterpreterData;
-
-typedef int (*crossinterpdatafunc)(PyObject *, _PyCrossInterpreterData *);
-PyAPI_FUNC(int) _PyObject_CheckCrossInterpreterData(PyObject *);
-
-PyAPI_FUNC(int) _PyObject_GetCrossInterpreterData(PyObject *, _PyCrossInterpreterData *);
-PyAPI_FUNC(PyObject *) _PyCrossInterpreterData_NewObject(_PyCrossInterpreterData *);
-PyAPI_FUNC(void) _PyCrossInterpreterData_Release(_PyCrossInterpreterData *);
-
 /* cross-interpreter data registry */
 
 /* For now we use a global registry of shareable classes.  An
    alternative would be to add a tp_* slot for a class's
    crossinterpdatafunc. It would be simpler and more efficient. */
 
-PyAPI_FUNC(int) _PyCrossInterpreterData_Register_Class(PyTypeObject *, crossinterpdatafunc);
-PyAPI_FUNC(crossinterpdatafunc) _PyCrossInterpreterData_Lookup(PyObject *);
-
 struct _xidregitem;
 
 struct _xidregitem {

Include/interpreteridobject.h

@@ -0,0 +1,17 @@
+#ifndef Py_INTERPRETERIDOBJECT_H
+#define Py_INTERPRETERIDOBJECT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_INTERPRETERIDOBJECT_H
+#  include  "cpython/interpreteridobject.h"
+#  undef Py_CPYTHON_INTERPRETERIDOBJECT_H
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERPRETERIDOBJECT_H */

Makefile.pre.in

@@ -391,6 +391,7 @@ OBJECT_OBJS=	\
 		Objects/floatobject.o \
 		Objects/frameobject.o \
 		Objects/funcobject.o \
+		Objects/interpreteridobject.o \
 		Objects/iterobject.o \
 		Objects/listobject.o \
 		Objects/longobject.o \
@@ -977,6 +978,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/funcobject.h \
 		$(srcdir)/Include/genobject.h \
 		$(srcdir)/Include/import.h \
+		$(srcdir)/Include/interpreteridobject.h \
 		$(srcdir)/Include/intrcheck.h \
 		$(srcdir)/Include/iterobject.h \
 		$(srcdir)/Include/listobject.h \
@@ -1039,6 +1041,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/abstract.h \
 		$(srcdir)/Include/cpython/coreconfig.h \
 		$(srcdir)/Include/cpython/dictobject.h \
+		$(srcdir)/Include/cpython/interpreteridobject.h \
 		$(srcdir)/Include/cpython/object.h \
 		$(srcdir)/Include/cpython/objimpl.h \
 		$(srcdir)/Include/cpython/pyerrors.h \

Modules/_xxsubinterpretersmodule.c

@@ -4,7 +4,7 @@
 
 #include "Python.h"
 #include "frameobject.h"
-#include "pycore_pystate.h"
+#include "interpreteridobject.h"
 
 
 static char *
@@ -31,38 +31,6 @@ _get_current(void)
     return _PyInterpreterState_Get();
 }
 
-static int64_t
-_coerce_id(PyObject *orig)
-{
-    PyObject *pyid = PyNumber_Long(orig);
-    if (pyid == NULL) {
-        if (PyErr_ExceptionMatches(PyExc_TypeError)) {
-            PyErr_Format(PyExc_TypeError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        else {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    int64_t id = PyLong_AsLongLong(pyid);
-    Py_DECREF(pyid);
-    if (id == -1 && PyErr_Occurred() != NULL) {
-        if (!PyErr_ExceptionMatches(PyExc_OverflowError)) {
-            PyErr_Format(PyExc_ValueError,
-                         "'id' must be a non-negative int, got %R", orig);
-        }
-        return -1;
-    }
-    if (id < 0) {
-        PyErr_Format(PyExc_ValueError,
-                     "'id' must be a non-negative int, got %R", orig);
-        return -1;
-    }
-    return id;
-}
-
 
 /* data-sharing-specific code ***********************************************/
 
@@ -71,6 +39,8 @@ struct _sharednsitem {
     _PyCrossInterpreterData data;
 };
 
+static void _sharednsitem_clear(struct _sharednsitem *);  // forward
+
 static int
 _sharednsitem_init(struct _sharednsitem *item, PyObject *key, PyObject *value)
 {
@@ -79,6 +49,7 @@ _sharednsitem_init(struct _sharednsitem *item, PyObject *key, PyObject *value)
         return -1;
     }
     if (_PyObject_GetCrossInterpreterData(value, &item->data) != 0) {
+        _sharednsitem_clear(item);
         return -1;
     }
     return 0;
@@ -89,6 +60,7 @@ _sharednsitem_clear(struct _sharednsitem *item)
 {
     if (item->name != NULL) {
         PyMem_Free(item->name);
+        item->name = NULL;
     }
     _PyCrossInterpreterData_Release(&item->data);
 }
@@ -1339,13 +1311,13 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj)
         return -1;
     }
     if (_PyObject_GetCrossInterpreterData(obj, data) != 0) {
-        PyMem_Free(data);
         PyThread_release_lock(mutex);
+        PyMem_Free(data);
         return -1;
     }
 
     // Add the data to the channel.
-    int res = _channel_add(chan, interp->id, data);
+    int res = _channel_add(chan, PyInterpreterState_GetID(interp), data);
     PyThread_release_lock(mutex);
     if (res != 0) {
         _PyCrossInterpreterData_Release(data);
@@ -1373,7 +1345,7 @@ _channel_recv(_channels *channels, int64_t id)
     // Past this point we are responsible for releasing the mutex.
 
     // Pop off the next item from the channel.
-    _PyCrossInterpreterData *data = _channel_next(chan, interp->id);
+    _PyCrossInterpreterData *data = _channel_next(chan, PyInterpreterState_GetID(interp));
     PyThread_release_lock(mutex);
     if (data == NULL) {
         if (!PyErr_Occurred()) {
@@ -1410,7 +1382,7 @@ _channel_drop(_channels *channels, int64_t id, int send, int recv)
     // Past this point we are responsible for releasing the mutex.
 
     // Close one or both of the two ends.
-    int res = _channel_close_interpreter(chan, interp->id, send-recv);
+    int res = _channel_close_interpreter(chan, PyInterpreterState_GetID(interp), send-recv);
     PyThread_release_lock(mutex);
     return res;
 }
@@ -1481,7 +1453,7 @@ channelid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
         cid = ((channelid *)id)->id;
     }
     else {
-        cid = _coerce_id(id);
+        cid = _Py_CoerceID(id);
         if (cid < 0) {
             return NULL;
         }
@@ -1875,7 +1847,7 @@ _run_script(PyInterpreterState *interp, const char *codestr,
     PyObject *excval = NULL;
     PyObject *tb = NULL;
 
-    PyObject *main_mod = PyMapping_GetItemString(interp->modules, "__main__");
+    PyObject *main_mod = _PyInterpreterState_GetMainModule(interp);
     if (main_mod == NULL) {
         goto error;
     }
@@ -1974,272 +1946,6 @@ _run_script_in_interpreter(PyInterpreterState *interp, const char *codestr,
     return result;
 }
 
-/* InterpreterID class */
-
-static PyTypeObject InterpreterIDtype;
-
-typedef struct interpid {
-    PyObject_HEAD
-    int64_t id;
-} interpid;
-
-static interpid *
-newinterpid(PyTypeObject *cls, int64_t id, int force)
-{
-    PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
-    if (interp == NULL) {
-        if (force) {
-            PyErr_Clear();
-        }
-        else {
-            return NULL;
-        }
-    }
-
-    interpid *self = PyObject_New(interpid, cls);
-    if (self == NULL) {
-        return NULL;
-    }
-    self->id = id;
-
-    if (interp != NULL) {
-        _PyInterpreterState_IDIncref(interp);
-    }
-    return self;
-}
-
-static PyObject *
-interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
-{
-    static char *kwlist[] = {"id", "force", NULL};
-    PyObject *idobj;
-    int force = 0;
-    if (!PyArg_ParseTupleAndKeywords(args, kwds,
-                                     "O|$p:InterpreterID.__init__", kwlist,
-                                     &idobj, &force)) {
-        return NULL;
-    }
-
-    // Coerce and check the ID.
-    int64_t id;
-    if (PyObject_TypeCheck(idobj, &InterpreterIDtype)) {
-        id = ((interpid *)idobj)->id;
-    }
-    else {
-        id = _coerce_id(idobj);
-        if (id < 0) {
-            return NULL;
-        }
-    }
-
-    return (PyObject *)newinterpid(cls, id, force);
-}
-
-static void
-interpid_dealloc(PyObject *v)
-{
-    int64_t id = ((interpid *)v)->id;
-    PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
-    if (interp != NULL) {
-        _PyInterpreterState_IDDecref(interp);
-    }
-    else {
-        // already deleted
-        PyErr_Clear();
-    }
-    Py_TYPE(v)->tp_free(v);
-}
-
-static PyObject *
-interpid_repr(PyObject *self)
-{
-    PyTypeObject *type = Py_TYPE(self);
-    const char *name = _PyType_Name(type);
-    interpid *id = (interpid *)self;
-    return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id);
-}
-
-static PyObject *
-interpid_str(PyObject *self)
-{
-    interpid *id = (interpid *)self;
-    return PyUnicode_FromFormat("%" PRId64 "", id->id);
-}
-
-PyObject *
-interpid_int(PyObject *self)
-{
-    interpid *id = (interpid *)self;
-    return PyLong_FromLongLong(id->id);
-}
-
-static PyNumberMethods interpid_as_number = {
-     0,                       /* nb_add */
-     0,                       /* nb_subtract */
-     0,                       /* nb_multiply */
-     0,                       /* nb_remainder */
-     0,                       /* nb_divmod */
-     0,                       /* nb_power */
-     0,                       /* nb_negative */
-     0,                       /* nb_positive */
-     0,                       /* nb_absolute */
-     0,                       /* nb_bool */
-     0,                       /* nb_invert */
-     0,                       /* nb_lshift */
-     0,                       /* nb_rshift */
-     0,                       /* nb_and */
-     0,                       /* nb_xor */
-     0,                       /* nb_or */
-     (unaryfunc)interpid_int, /* nb_int */
-     0,                       /* nb_reserved */
-     0,                       /* nb_float */
-
-     0,                       /* nb_inplace_add */
-     0,                       /* nb_inplace_subtract */
-     0,                       /* nb_inplace_multiply */
-     0,                       /* nb_inplace_remainder */
-     0,                       /* nb_inplace_power */
-     0,                       /* nb_inplace_lshift */
-     0,                       /* nb_inplace_rshift */
-     0,                       /* nb_inplace_and */
-     0,                       /* nb_inplace_xor */
-     0,                       /* nb_inplace_or */
-
-     0,                       /* nb_floor_divide */
-     0,                       /* nb_true_divide */
-     0,                       /* nb_inplace_floor_divide */
-     0,                       /* nb_inplace_true_divide */
-
-     (unaryfunc)interpid_int, /* nb_index */
-};
-
-static Py_hash_t
-interpid_hash(PyObject *self)
-{
-    interpid *id = (interpid *)self;
-    PyObject *obj = PyLong_FromLongLong(id->id);
-    if (obj == NULL) {
-        return -1;
-    }
-    Py_hash_t hash = PyObject_Hash(obj);
-    Py_DECREF(obj);
-    return hash;
-}
-
-static PyObject *
-interpid_richcompare(PyObject *self, PyObject *other, int op)
-{
-    if (op != Py_EQ && op != Py_NE) {
-        Py_RETURN_NOTIMPLEMENTED;
-    }
-
-    if (!PyObject_TypeCheck(self, &InterpreterIDtype)) {
-        Py_RETURN_NOTIMPLEMENTED;
-    }
-
-    interpid *id = (interpid *)self;
-    int equal;
-    if (PyObject_TypeCheck(other, &InterpreterIDtype)) {
-        interpid *otherid = (interpid *)other;
-        equal = (id->id == otherid->id);
-    }
-    else {
-        other = PyNumber_Long(other);
-        if (other == NULL) {
-            PyErr_Clear();
-            Py_RETURN_NOTIMPLEMENTED;
-        }
-        int64_t otherid = PyLong_AsLongLong(other);
-        Py_DECREF(other);
-        if (otherid == -1 && PyErr_Occurred() != NULL) {
-            return NULL;
-        }
-        if (otherid < 0) {
-            equal = 0;
-        }
-        else {
-            equal = (id->id == otherid);
-        }
-    }
-
-    if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
-        Py_RETURN_TRUE;
-    }
-    Py_RETURN_FALSE;
-}
-
-PyDoc_STRVAR(interpid_doc,
-"A interpreter ID identifies a interpreter and may be used as an int.");
-
-static PyTypeObject InterpreterIDtype = {
-    PyVarObject_HEAD_INIT(&PyType_Type, 0)
-    "interpreters.InterpreterID",   /* tp_name */
-    sizeof(interpid),               /* tp_basicsize */
-    0,                              /* tp_itemsize */
-    (destructor)interpid_dealloc,   /* tp_dealloc */
-    0,                              /* tp_print */
-    0,                              /* tp_getattr */
-    0,                              /* tp_setattr */
-    0,                              /* tp_as_async */
-    (reprfunc)interpid_repr,        /* tp_repr */
-    &interpid_as_number,            /* tp_as_number */
-    0,                              /* tp_as_sequence */
-    0,                              /* tp_as_mapping */
-    interpid_hash,                  /* tp_hash */
-    0,                              /* tp_call */
-    (reprfunc)interpid_str,         /* tp_str */
-    0,                              /* tp_getattro */
-    0,                              /* tp_setattro */
-    0,                              /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
-        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
-    interpid_doc,                   /* tp_doc */
-    0,                              /* tp_traverse */
-    0,                              /* tp_clear */
-    interpid_richcompare,           /* tp_richcompare */
-    0,                              /* tp_weaklistoffset */
-    0,                              /* tp_iter */
-    0,                              /* tp_iternext */
-    0,                              /* tp_methods */
-    0,                              /* tp_members */
-    0,                              /* tp_getset */
-    0,                              /* tp_base */
-    0,                              /* tp_dict */
-    0,                              /* tp_descr_get */
-    0,                              /* tp_descr_set */
-    0,                              /* tp_dictoffset */
-    0,                              /* tp_init */
-    0,                              /* tp_alloc */
-    interpid_new,                   /* tp_new */
-};
-
-static PyObject *
-_get_id(PyInterpreterState *interp)
-{
-    PY_INT64_T id = PyInterpreterState_GetID(interp);
-    if (id < 0) {
-        return NULL;
-    }
-    return (PyObject *)newinterpid(&InterpreterIDtype, id, 0);
-}
-
-static PyInterpreterState *
-_look_up(PyObject *requested_id)
-{
-    int64_t id;
-    if (PyObject_TypeCheck(requested_id, &InterpreterIDtype)) {
-        id = ((interpid *)requested_id)->id;
-    }
-    else {
-        id = PyLong_AsLongLong(requested_id);
-        if (id == -1 && PyErr_Occurred() != NULL) {
-            return NULL;
-        }
-        assert(id <= INT64_MAX);
-    }
-    return _PyInterpreterState_LookUpID(id);
-}
-
 
 /* module level code ********************************************************/
 
@@ -2283,17 +1989,16 @@ interp_create(PyObject *self, PyObject *args)
         PyErr_SetString(PyExc_RuntimeError, "interpreter creation failed");
         return NULL;
     }
-    if (_PyInterpreterState_IDInitref(tstate->interp) != 0) {
+    PyObject *idobj = _PyInterpreterState_GetIDObject(tstate->interp);
-        goto error;
+    if (idobj == NULL) {
-    };
+        // XXX Possible GILState issues?
-    return _get_id(tstate->interp);
+        save_tstate = PyThreadState_Swap(tstate);
-
+        Py_EndInterpreter(tstate);
-error:
+        PyThreadState_Swap(save_tstate);
-    // XXX Possible GILState issues?
+        return NULL;
-    save_tstate = PyThreadState_Swap(tstate);
+    }
-    Py_EndInterpreter(tstate);
+    _PyInterpreterState_RequireIDRef(tstate->interp, 1);
-    PyThreadState_Swap(save_tstate);
+    return idobj;
-    return NULL;
 }
 
 PyDoc_STRVAR(create_doc,
@@ -2318,7 +2023,7 @@ interp_destroy(PyObject *self, PyObject *args, PyObject *kwds)
     }
 
     // Look up the interpreter.
-    PyInterpreterState *interp = _look_up(id);
+    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -2374,7 +2079,7 @@ interp_list_all(PyObject *self, PyObject *Py_UNUSED(ignored))
 
     interp = PyInterpreterState_Head();
     while (interp != NULL) {
-        id = _get_id(interp);
+        id = _PyInterpreterState_GetIDObject(interp);
         if (id == NULL) {
             Py_DECREF(ids);
             return NULL;
@@ -2406,7 +2111,7 @@ interp_get_current(PyObject *self, PyObject *Py_UNUSED(ignored))
     if (interp == NULL) {
         return NULL;
     }
-    return _get_id(interp);
+    return _PyInterpreterState_GetIDObject(interp);
 }
 
 PyDoc_STRVAR(get_current_doc,
@@ -2420,7 +2125,7 @@ interp_get_main(PyObject *self, PyObject *Py_UNUSED(ignored))
 {
     // Currently, 0 is always the main interpreter.
     PY_INT64_T id = 0;
-    return (PyObject *)newinterpid(&InterpreterIDtype, id, 0);
+    return _PyInterpreterID_New(id);
 }
 
 PyDoc_STRVAR(get_main_doc,
@@ -2446,7 +2151,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds)
     }
 
     // Look up the interpreter.
-    PyInterpreterState *interp = _look_up(id);
+    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -2516,7 +2221,7 @@ interp_is_running(PyObject *self, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyInterpreterState *interp = _look_up(id);
+    PyInterpreterState *interp = _PyInterpreterID_LookUp(id);
     if (interp == NULL) {
         return NULL;
     }
@@ -2568,7 +2273,7 @@ channel_destroy(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:channel_destroy", kwlist, &id)) {
         return NULL;
     }
-    int64_t cid = _coerce_id(id);
+    int64_t cid = _Py_CoerceID(id);
     if (cid < 0) {
         return NULL;
     }
@@ -2632,7 +2337,7 @@ channel_send(PyObject *self, PyObject *args, PyObject *kwds)
                                      "OO:channel_send", kwlist, &id, &obj)) {
         return NULL;
     }
-    int64_t cid = _coerce_id(id);
+    int64_t cid = _Py_CoerceID(id);
     if (cid < 0) {
         return NULL;
     }
@@ -2657,7 +2362,7 @@ channel_recv(PyObject *self, PyObject *args, PyObject *kwds)
                                      "O:channel_recv", kwlist, &id)) {
         return NULL;
     }
-    int64_t cid = _coerce_id(id);
+    int64_t cid = _Py_CoerceID(id);
     if (cid < 0) {
         return NULL;
     }
@@ -2683,7 +2388,7 @@ channel_close(PyObject *self, PyObject *args, PyObject *kwds)
                                      &id, &send, &recv, &force)) {
         return NULL;
     }
-    int64_t cid = _coerce_id(id);
+    int64_t cid = _Py_CoerceID(id);
     if (cid < 0) {
         return NULL;
     }
@@ -2735,7 +2440,7 @@ channel_release(PyObject *self, PyObject *args, PyObject *kwds)
                                      &id, &send, &recv, &force)) {
         return NULL;
     }
-    int64_t cid = _coerce_id(id);
+    int64_t cid = _Py_CoerceID(id);
     if (cid < 0) {
         return NULL;
     }
@@ -2837,10 +2542,6 @@ PyInit__xxsubinterpreters(void)
     if (PyType_Ready(&ChannelIDtype) != 0) {
         return NULL;
     }
-    InterpreterIDtype.tp_base = &PyLong_Type;
-    if (PyType_Ready(&InterpreterIDtype) != 0) {
-        return NULL;
-    }
 
     /* Create the module */
     PyObject *module = PyModule_Create(&interpretersmodule);
@@ -2862,12 +2563,12 @@ PyInit__xxsubinterpreters(void)
     if (PyDict_SetItemString(ns, "ChannelID", (PyObject *)&ChannelIDtype) != 0) {
         return NULL;
     }
-    Py_INCREF(&InterpreterIDtype);
+    Py_INCREF(&_PyInterpreterID_Type);
-    if (PyDict_SetItemString(ns, "InterpreterID", (PyObject *)&InterpreterIDtype) != 0) {
+    if (PyDict_SetItemString(ns, "InterpreterID", (PyObject *)&_PyInterpreterID_Type) != 0) {
         return NULL;
     }
 
-    if (_PyCrossInterpreterData_Register_Class(&ChannelIDtype, _channelid_shared)) {
+    if (_PyCrossInterpreterData_RegisterClass(&ChannelIDtype, _channelid_shared)) {
         return NULL;
     }
 

Objects/interpreteridobject.c

@@ -0,0 +1,308 @@
+/* InterpreterID object */
+
+#include "Python.h"
+#include "internal/pycore_pystate.h"
+#include "interpreteridobject.h"
+
+
+int64_t
+_Py_CoerceID(PyObject *orig)
+{
+    PyObject *pyid = PyNumber_Long(orig);
+    if (pyid == NULL) {
+        if (PyErr_ExceptionMatches(PyExc_TypeError)) {
+            PyErr_Format(PyExc_TypeError,
+                         "'id' must be a non-negative int, got %R", orig);
+        }
+        else {
+            PyErr_Format(PyExc_ValueError,
+                         "'id' must be a non-negative int, got %R", orig);
+        }
+        return -1;
+    }
+    int64_t id = PyLong_AsLongLong(pyid);
+    Py_DECREF(pyid);
+    if (id == -1 && PyErr_Occurred() != NULL) {
+        if (!PyErr_ExceptionMatches(PyExc_OverflowError)) {
+            PyErr_Format(PyExc_ValueError,
+                         "'id' must be a non-negative int, got %R", orig);
+        }
+        return -1;
+    }
+    if (id < 0) {
+        PyErr_Format(PyExc_ValueError,
+                     "'id' must be a non-negative int, got %R", orig);
+        return -1;
+    }
+    return id;
+}
+
+typedef struct interpid {
+    PyObject_HEAD
+    int64_t id;
+} interpid;
+
+static interpid *
+newinterpid(PyTypeObject *cls, int64_t id, int force)
+{
+    PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
+    if (interp == NULL) {
+        if (force) {
+            PyErr_Clear();
+        }
+        else {
+            return NULL;
+        }
+    }
+
+    interpid *self = PyObject_New(interpid, cls);
+    if (self == NULL) {
+        return NULL;
+    }
+    self->id = id;
+
+    if (interp != NULL) {
+        _PyInterpreterState_IDIncref(interp);
+    }
+    return self;
+}
+
+static PyObject *
+interpid_new(PyTypeObject *cls, PyObject *args, PyObject *kwds)
+{
+    static char *kwlist[] = {"id", "force", NULL};
+    PyObject *idobj;
+    int force = 0;
+    if (!PyArg_ParseTupleAndKeywords(args, kwds,
+                                     "O|$p:InterpreterID.__init__", kwlist,
+                                     &idobj, &force)) {
+        return NULL;
+    }
+
+    // Coerce and check the ID.
+    int64_t id;
+    if (PyObject_TypeCheck(idobj, &_PyInterpreterID_Type)) {
+        id = ((interpid *)idobj)->id;
+    }
+    else {
+        id = _Py_CoerceID(idobj);
+        if (id < 0) {
+            return NULL;
+        }
+    }
+
+    return (PyObject *)newinterpid(cls, id, force);
+}
+
+static void
+interpid_dealloc(PyObject *v)
+{
+    int64_t id = ((interpid *)v)->id;
+    PyInterpreterState *interp = _PyInterpreterState_LookUpID(id);
+    if (interp != NULL) {
+        _PyInterpreterState_IDDecref(interp);
+    }
+    else {
+        // already deleted
+        PyErr_Clear();
+    }
+    Py_TYPE(v)->tp_free(v);
+}
+
+static PyObject *
+interpid_repr(PyObject *self)
+{
+    PyTypeObject *type = Py_TYPE(self);
+    const char *name = _PyType_Name(type);
+    interpid *id = (interpid *)self;
+    return PyUnicode_FromFormat("%s(%" PRId64 ")", name, id->id);
+}
+
+static PyObject *
+interpid_str(PyObject *self)
+{
+    interpid *id = (interpid *)self;
+    return PyUnicode_FromFormat("%" PRId64 "", id->id);
+}
+
+static PyObject *
+interpid_int(PyObject *self)
+{
+    interpid *id = (interpid *)self;
+    return PyLong_FromLongLong(id->id);
+}
+
+static PyNumberMethods interpid_as_number = {
+     0,                       /* nb_add */
+     0,                       /* nb_subtract */
+     0,                       /* nb_multiply */
+     0,                       /* nb_remainder */
+     0,                       /* nb_divmod */
+     0,                       /* nb_power */
+     0,                       /* nb_negative */
+     0,                       /* nb_positive */
+     0,                       /* nb_absolute */
+     0,                       /* nb_bool */
+     0,                       /* nb_invert */
+     0,                       /* nb_lshift */
+     0,                       /* nb_rshift */
+     0,                       /* nb_and */
+     0,                       /* nb_xor */
+     0,                       /* nb_or */
+     (unaryfunc)interpid_int, /* nb_int */
+     0,                       /* nb_reserved */
+     0,                       /* nb_float */
+
+     0,                       /* nb_inplace_add */
+     0,                       /* nb_inplace_subtract */
+     0,                       /* nb_inplace_multiply */
+     0,                       /* nb_inplace_remainder */
+     0,                       /* nb_inplace_power */
+     0,                       /* nb_inplace_lshift */
+     0,                       /* nb_inplace_rshift */
+     0,                       /* nb_inplace_and */
+     0,                       /* nb_inplace_xor */
+     0,                       /* nb_inplace_or */
+
+     0,                       /* nb_floor_divide */
+     0,                       /* nb_true_divide */
+     0,                       /* nb_inplace_floor_divide */
+     0,                       /* nb_inplace_true_divide */
+
+     (unaryfunc)interpid_int, /* nb_index */
+};
+
+static Py_hash_t
+interpid_hash(PyObject *self)
+{
+    interpid *id = (interpid *)self;
+    PyObject *obj = PyLong_FromLongLong(id->id);
+    if (obj == NULL) {
+        return -1;
+    }
+    Py_hash_t hash = PyObject_Hash(obj);
+    Py_DECREF(obj);
+    return hash;
+}
+
+static PyObject *
+interpid_richcompare(PyObject *self, PyObject *other, int op)
+{
+    if (op != Py_EQ && op != Py_NE) {
+        Py_RETURN_NOTIMPLEMENTED;
+    }
+
+    if (!PyObject_TypeCheck(self, &_PyInterpreterID_Type)) {
+        Py_RETURN_NOTIMPLEMENTED;
+    }
+
+    interpid *id = (interpid *)self;
+    int equal;
+    if (PyObject_TypeCheck(other, &_PyInterpreterID_Type)) {
+        interpid *otherid = (interpid *)other;
+        equal = (id->id == otherid->id);
+    }
+    else {
+        other = PyNumber_Long(other);
+        if (other == NULL) {
+            PyErr_Clear();
+            Py_RETURN_NOTIMPLEMENTED;
+        }
+        int64_t otherid = PyLong_AsLongLong(other);
+        Py_DECREF(other);
+        if (otherid == -1 && PyErr_Occurred() != NULL) {
+            return NULL;
+        }
+        if (otherid < 0) {
+            equal = 0;
+        }
+        else {
+            equal = (id->id == otherid);
+        }
+    }
+
+    if ((op == Py_EQ && equal) || (op == Py_NE && !equal)) {
+        Py_RETURN_TRUE;
+    }
+    Py_RETURN_FALSE;
+}
+
+PyDoc_STRVAR(interpid_doc,
+"A interpreter ID identifies a interpreter and may be used as an int.");
+
+PyTypeObject _PyInterpreterID_Type = {
+    PyVarObject_HEAD_INIT(&PyType_Type, 0)
+    "InterpreterID",   /* tp_name */
+    sizeof(interpid),               /* tp_basicsize */
+    0,                              /* tp_itemsize */
+    (destructor)interpid_dealloc,   /* tp_dealloc */
+    0,                              /* tp_print */
+    0,                              /* tp_getattr */
+    0,                              /* tp_setattr */
+    0,                              /* tp_as_async */
+    (reprfunc)interpid_repr,        /* tp_repr */
+    &interpid_as_number,            /* tp_as_number */
+    0,                              /* tp_as_sequence */
+    0,                              /* tp_as_mapping */
+    interpid_hash,                  /* tp_hash */
+    0,                              /* tp_call */
+    (reprfunc)interpid_str,         /* tp_str */
+    0,                              /* tp_getattro */
+    0,                              /* tp_setattro */
+    0,                              /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE |
+        Py_TPFLAGS_LONG_SUBCLASS,   /* tp_flags */
+    interpid_doc,                   /* tp_doc */
+    0,                              /* tp_traverse */
+    0,                              /* tp_clear */
+    interpid_richcompare,           /* tp_richcompare */
+    0,                              /* tp_weaklistoffset */
+    0,                              /* tp_iter */
+    0,                              /* tp_iternext */
+    0,                              /* tp_methods */
+    0,                              /* tp_members */
+    0,                              /* tp_getset */
+    &PyLong_Type,                   /* tp_base */
+    0,                              /* tp_dict */
+    0,                              /* tp_descr_get */
+    0,                              /* tp_descr_set */
+    0,                              /* tp_dictoffset */
+    0,                              /* tp_init */
+    0,                              /* tp_alloc */
+    interpid_new,                   /* tp_new */
+};
+
+PyObject *_PyInterpreterID_New(int64_t id)
+{
+    return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0);
+}
+
+PyObject *
+_PyInterpreterState_GetIDObject(PyInterpreterState *interp)
+{
+    if (_PyInterpreterState_IDInitref(interp) != 0) {
+        return NULL;
+    };
+    PY_INT64_T id = PyInterpreterState_GetID(interp);
+    if (id < 0) {
+        return NULL;
+    }
+    return (PyObject *)newinterpid(&_PyInterpreterID_Type, id, 0);
+}
+
+PyInterpreterState *
+_PyInterpreterID_LookUp(PyObject *requested_id)
+{
+    int64_t id;
+    if (PyObject_TypeCheck(requested_id, &_PyInterpreterID_Type)) {
+        id = ((interpid *)requested_id)->id;
+    }
+    else {
+        id = PyLong_AsLongLong(requested_id);
+        if (id == -1 && PyErr_Occurred() != NULL) {
+            return NULL;
+        }
+        assert(id <= INT64_MAX);
+    }
+    return _PyInterpreterState_LookUpID(id);
+}

Objects/object.c

@@ -5,6 +5,7 @@
 #include "pycore_pystate.h"
 #include "pycore_context.h"
 #include "frameobject.h"
+#include "interpreteridobject.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -1806,6 +1807,7 @@ _PyTypes_Init(void)
     INIT_TYPE(&PySeqIter_Type, "sequence iterator");
     INIT_TYPE(&PyCoro_Type, "coroutine");
     INIT_TYPE(&_PyCoroWrapper_Type, "coroutine wrapper");
+    INIT_TYPE(&_PyInterpreterID_Type, "interpreter ID");
     return _Py_INIT_OK();
 
 #undef INIT_TYPE

PCbuild/pythoncore.vcxproj

@@ -154,6 +154,7 @@
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
     <ClInclude Include="..\Include\internal\pycore_tupleobject.h" />
     <ClInclude Include="..\Include\internal\pycore_warnings.h" />
+    <ClInclude Include="..\Include\interpreteridobject.h" />
     <ClInclude Include="..\Include\intrcheck.h" />
     <ClInclude Include="..\Include\iterobject.h" />
     <ClInclude Include="..\Include\listobject.h" />
@@ -350,6 +351,7 @@
     <ClCompile Include="..\Objects\frameobject.c" />
     <ClCompile Include="..\Objects\funcobject.c" />
     <ClCompile Include="..\Objects\genobject.c" />
+    <ClCompile Include="..\Objects\interpreteridobject.c" />
     <ClCompile Include="..\Objects\iterobject.c" />
     <ClCompile Include="..\Objects\listobject.c" />
     <ClCompile Include="..\Objects\longobject.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -483,6 +483,9 @@
     <ClInclude Include="..\Include\namespaceobject.h">
       <Filter>Include</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\interpreteridobject.h">
+      <Filter>Include</Filter>
+    </ClInclude>
     <ClInclude Include="..\Modules\hashtable.h">
       <Filter>Modules</Filter>
     </ClInclude>
@@ -1043,6 +1046,9 @@
     <ClCompile Include="..\Objects\namespaceobject.c">
       <Filter>Objects</Filter>
     </ClCompile>
+    <ClCompile Include="..\Objects\interpreteridobject.c">
+      <Filter>Objects</Filter>
+    </ClCompile>
     <ClCompile Include="..\Modules\_opcode.c">
       <Filter>Modules</Filter>
     </ClCompile>

Python/pystate.c

@@ -403,7 +403,7 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp)
     int64_t refcount = interp->id_refcount;
     PyThread_release_lock(interp->id_mutex);
 
-    if (refcount == 0) {
+    if (refcount == 0 && interp->requires_idref) {
         // XXX Using the "head" thread isn't strictly correct.
         PyThreadState *tstate = PyInterpreterState_ThreadHead(interp);
         // XXX Possible GILState issues?
@@ -413,6 +413,18 @@ _PyInterpreterState_IDDecref(PyInterpreterState *interp)
     }
 }
 
+int
+_PyInterpreterState_RequiresIDRef(PyInterpreterState *interp)
+{
+    return interp->requires_idref;
+}
+
+void
+_PyInterpreterState_RequireIDRef(PyInterpreterState *interp, int required)
+{
+    interp->requires_idref = required ? 1 : 0;
+}
+
 _PyCoreConfig *
 _PyInterpreterState_GetCoreConfig(PyInterpreterState *interp)
 {
@@ -425,6 +437,16 @@ _PyInterpreterState_GetMainConfig(PyInterpreterState *interp)
     return &interp->config;
 }
 
+PyObject *
+_PyInterpreterState_GetMainModule(PyInterpreterState *interp)
+{
+    if (interp->modules == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "interpreter not initialized");
+        return NULL;
+    }
+    return PyMapping_GetItemString(interp->modules, "__main__");
+}
+
 /* Default implementation for _PyThreadState_GetFrame */
 static struct _frame *
 threadstate_getframe(PyThreadState *self)
@@ -1370,7 +1392,7 @@ _register_xidata(PyTypeObject *cls, crossinterpdatafunc getdata)
 static void _register_builtins_for_crossinterpreter_data(void);
 
 int
-_PyCrossInterpreterData_Register_Class(PyTypeObject *cls,
+_PyCrossInterpreterData_RegisterClass(PyTypeObject *cls,
                                        crossinterpdatafunc getdata)
 {
     if (!PyType_Check(cls)) {

setup.py

@@ -784,9 +784,7 @@ class PyBuildExt(build_ext):
         self.add(Extension('syslog', ['syslogmodule.c']))
 
         # Python interface to subinterpreter C-API.
-        self.add(Extension('_xxsubinterpreters',
+        self.add(Extension('_xxsubinterpreters', ['_xxsubinterpretersmodule.c']))
-                           ['_xxsubinterpretersmodule.c'],
-                           define_macros=[('Py_BUILD_CORE', '')]))
 
         #
         # Here ends the simple stuff.  From here on, modules need certain