gh-98627: Add an Optional Check for Extension Module Subinterpreter Compatibility (gh-99040)

Enforcing (optionally) the restriction set by PEP 489 makes sense. Furthermore, this sets the stage for a potential restriction related to a per-interpreter GIL.

This change includes the following:

* add tests for extension module subinterpreter compatibility
* add _PyInterpreterConfig.check_multi_interp_extensions
* add Py_RTFLAGS_MULTI_INTERP_EXTENSIONS
* add _PyImport_CheckSubinterpIncompatibleExtensionAllowed()
* fail iff the module does not implement multi-phase init and the current interpreter is configured to check

https://github.com/python/cpython/issues/98627

Python/clinic/import.c.h

@@ -442,6 +442,37 @@ exit:
     return return_value;
 }
 
+PyDoc_STRVAR(_imp__override_multi_interp_extensions_check__doc__,
+"_override_multi_interp_extensions_check($module, override, /)\n"
+"--\n"
+"\n"
+"(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.\n"
+"\n"
+"(-1: \"never\", 1: \"always\", 0: no override)");
+
+#define _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF    \
+    {"_override_multi_interp_extensions_check", (PyCFunction)_imp__override_multi_interp_extensions_check, METH_O, _imp__override_multi_interp_extensions_check__doc__},
+
+static PyObject *
+_imp__override_multi_interp_extensions_check_impl(PyObject *module,
+                                                  int override);
+
+static PyObject *
+_imp__override_multi_interp_extensions_check(PyObject *module, PyObject *arg)
+{
+    PyObject *return_value = NULL;
+    int override;
+
+    override = _PyLong_AsInt(arg);
+    if (override == -1 && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = _imp__override_multi_interp_extensions_check_impl(module, override);
+
+exit:
+    return return_value;
+}
+
 #if defined(HAVE_DYNAMIC_LOADING)
 
 PyDoc_STRVAR(_imp_create_dynamic__doc__,
@@ -617,4 +648,4 @@ exit:
 #ifndef _IMP_EXEC_DYNAMIC_METHODDEF
     #define _IMP_EXEC_DYNAMIC_METHODDEF
 #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */
-/*[clinic end generated code: output=806352838c3f7008 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=b18d46e0036eff49 input=a9049054013a1b77]*/

Python/import.c

@@ -74,6 +74,8 @@ static struct _inittab *inittab_copy = NULL;
     (interp)->imports.modules_by_index
 #define IMPORTLIB(interp) \
     (interp)->imports.importlib
+#define OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) \
+    (interp)->imports.override_multi_interp_extensions_check
 #define OVERRIDE_FROZEN_MODULES(interp) \
     (interp)->imports.override_frozen_modules
 #ifdef HAVE_DLOPEN
@@ -816,6 +818,38 @@ _extensions_cache_clear_all(void)
     Py_CLEAR(EXTENSIONS);
 }
 
+
+static bool
+check_multi_interp_extensions(PyInterpreterState *interp)
+{
+    int override = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp);
+    if (override < 0) {
+        return false;
+    }
+    else if (override > 0) {
+        return true;
+    }
+    else if (_PyInterpreterState_HasFeature(
+                interp, Py_RTFLAGS_MULTI_INTERP_EXTENSIONS)) {
+        return true;
+    }
+    return false;
+}
+
+int
+_PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name)
+{
+    PyInterpreterState *interp = _PyInterpreterState_Get();
+    if (check_multi_interp_extensions(interp)) {
+        assert(!_Py_IsMainInterpreter(interp));
+        PyErr_Format(PyExc_ImportError,
+                     "module %s does not support loading in subinterpreters",
+                     name);
+        return -1;
+    }
+    return 0;
+}
+
 static int
 fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename)
 {
@@ -3297,6 +3331,34 @@ _imp__override_frozen_modules_for_tests_impl(PyObject *module, int override)
     Py_RETURN_NONE;
 }
 
+/*[clinic input]
+_imp._override_multi_interp_extensions_check
+
+    override: int
+    /
+
+(internal-only) Override PyInterpreterConfig.check_multi_interp_extensions.
+
+(-1: "never", 1: "always", 0: no override)
+[clinic start generated code]*/
+
+static PyObject *
+_imp__override_multi_interp_extensions_check_impl(PyObject *module,
+                                                  int override)
+/*[clinic end generated code: output=3ff043af52bbf280 input=e086a2ea181f92ae]*/
+{
+    PyInterpreterState *interp = _PyInterpreterState_GET();
+    if (_Py_IsMainInterpreter(interp)) {
+        PyErr_SetString(PyExc_RuntimeError,
+                        "_imp._override_multi_interp_extensions_check() "
+                        "cannot be used in the main interpreter");
+        return NULL;
+    }
+    int oldvalue = OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp);
+    OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK(interp) = override;
+    return PyLong_FromLong(oldvalue);
+}
+
 #ifdef HAVE_DYNAMIC_LOADING
 
 /*[clinic input]
@@ -3329,18 +3391,23 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file)
 
     PyThreadState *tstate = _PyThreadState_GET();
     mod = import_find_extension(tstate, name, path);
-    if (mod != NULL || PyErr_Occurred()) {
-        Py_DECREF(name);
-        Py_DECREF(path);
-        return mod;
+    if (mod != NULL) {
+        const char *name_buf = PyUnicode_AsUTF8(name);
+        assert(name_buf != NULL);
+        if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) {
+            Py_DECREF(mod);
+            mod = NULL;
+        }
+        goto finally;
+    }
+    else if (PyErr_Occurred()) {
+        goto finally;
     }
 
     if (file != NULL) {
         fp = _Py_fopen_obj(path, "r");
         if (fp == NULL) {
-            Py_DECREF(name);
-            Py_DECREF(path);
-            return NULL;
+            goto finally;
         }
     }
     else
@@ -3348,10 +3415,12 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file)
 
     mod = _PyImport_LoadDynamicModuleWithSpec(spec, fp);
 
-    Py_DECREF(name);
-    Py_DECREF(path);
     if (fp)
         fclose(fp);
+
+finally:
+    Py_DECREF(name);
+    Py_DECREF(path);
     return mod;
 }
 
@@ -3436,6 +3505,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_IS_FROZEN_METHODDEF
     _IMP__FROZEN_MODULE_NAMES_METHODDEF
     _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF
+    _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF

Python/importdl.c

@@ -3,6 +3,7 @@
 
 #include "Python.h"
 #include "pycore_call.h"
+#include "pycore_import.h"
 #include "pycore_pystate.h"
 #include "pycore_runtime.h"
 
@@ -203,6 +204,10 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
 
     /* Fall back to single-phase init mechanism */
 
+    if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) {
+        goto error;
+    }
+
     if (hook_prefix == nonascii_prefix) {
         /* don't allow legacy init for non-ASCII module names */
         PyErr_Format(

Python/pylifecycle.c

@@ -565,6 +565,10 @@ init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *con
     if (config->allow_daemon_threads) {
         interp->feature_flags |= Py_RTFLAGS_DAEMON_THREADS;
     }
+
+    if (config->check_multi_interp_extensions) {
+        interp->feature_flags |= Py_RTFLAGS_MULTI_INTERP_EXTENSIONS;
+    }
 }