bpo-31404: Revert "remove modules from Py_InterpreterState (#1638)" (#3565)

PR #1638, for bpo-28411, causes problems in some (very) edge cases. Until that gets sorted out, we're reverting the merge. PR #3506, a fix on top of #1638, is also getting reverted.

Doc/c-api/import.rst

@@ -204,13 +204,6 @@ Importing Modules
    Return the dictionary used for the module administration (a.k.a.
    ``sys.modules``).  Note that this is a per-interpreter variable.
 
-.. c:function:: PyObject* PyImport_GetModule(PyObject *name)
-
-   Return the already imported module with the given name.  If the
-   module has not been imported yet then returns NULL but does not set
-   an error.  Returns NULL and sets an error if the lookup failed.
-
-   .. versionadded:: 3.7
 
 .. c:function:: PyObject* PyImport_GetImporter(PyObject *path)
 

Doc/whatsnew/3.7.rst

@@ -486,9 +486,6 @@ Changes in the Python API
   and module are affected by this change. (Contributed by INADA Naoki and
   Eugene Toder in :issue:`29463`.)
 
-* ``PyInterpreterState`` no longer has a ``modules`` field.  Instead use
-  ``sys.modules``.
-
 * The *mode* argument of :func:`os.makedirs` no longer affects the file
   permission bits of newly-created intermediate-level directories.
   To set their file permission bits you can set the umask before invoking

Include/import.h

@@ -38,25 +38,11 @@ PyAPI_FUNC(PyObject *) PyImport_ExecCodeModuleObject(
     );
 #endif
 PyAPI_FUNC(PyObject *) PyImport_GetModuleDict(void);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(int) _PyImport_IsInitialized(PyInterpreterState *);
-#endif
-PyAPI_FUNC(PyObject *) PyImport_GetModule(PyObject *name);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyImport_GetModule(PyObject *name);
-PyAPI_FUNC(PyObject *) _PyImport_GetModuleWithError(PyObject *name);
-PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(struct _Py_Identifier *name);
-PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
-PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module);
-#endif
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000
 PyAPI_FUNC(PyObject *) PyImport_AddModuleObject(
     PyObject *name
     );
 #endif
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyImport_AddModuleObject(PyObject *, PyObject *);
-#endif
 PyAPI_FUNC(PyObject *) PyImport_AddModule(
     const char *name            /* UTF-8 encoded string */
     );
@@ -106,19 +92,14 @@ PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
 PyAPI_FUNC(void) _PyImport_ReInitLock(void);
 
 PyAPI_FUNC(PyObject *) _PyImport_FindBuiltin(
-    const char *name,            /* UTF-8 encoded string */
+    const char *name            /* UTF-8 encoded string */
-    PyObject *modules
     );
 PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObject(PyObject *, PyObject *);
-PyAPI_FUNC(PyObject *) _PyImport_FindExtensionObjectEx(PyObject *, PyObject *,
-                                                       PyObject *);
 PyAPI_FUNC(int) _PyImport_FixupBuiltin(
     PyObject *mod,
-    const char *name,            /* UTF-8 encoded string */
+    const char *name            /* UTF-8 encoded string */
-    PyObject *modules
     );
-PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *,
+PyAPI_FUNC(int) _PyImport_FixupExtensionObject(PyObject*, PyObject *, PyObject *);
-                                               PyObject *, PyObject *);
 
 struct _inittab {
     const char *name;           /* ASCII encoded string */

Include/modsupport.h

@@ -191,10 +191,6 @@ PyAPI_FUNC(int) PyModule_ExecDef(PyObject *module, PyModuleDef *def);
 
 PyAPI_FUNC(PyObject *) PyModule_Create2(struct PyModuleDef*,
                                      int apiver);
-#ifndef Py_LIMITED_API
-PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(struct PyModuleDef*,
-                                                   int apiver);
-#endif
 
 #ifdef Py_LIMITED_API
 #define PyModule_Create(module) \

Include/object.h

@@ -727,13 +727,14 @@ PyAPI_FUNC(Py_ssize_t) _Py_GetRefTotal(void);
 /* Py_REF_DEBUG also controls the display of refcounts and memory block
  * allocations at the interactive prompt and at interpreter shutdown
  */
-PyAPI_FUNC(PyObject *) _PyDebug_XOptionShowRefCount(void);
 PyAPI_FUNC(void) _PyDebug_PrintTotalRefs(void);
+#define _PY_DEBUG_PRINT_TOTAL_REFS() _PyDebug_PrintTotalRefs()
 #else
 #define _Py_INC_REFTOTAL
 #define _Py_DEC_REFTOTAL
 #define _Py_REF_DEBUG_COMMA
 #define _Py_CHECK_REFCNT(OP)    /* a semicolon */;
+#define _PY_DEBUG_PRINT_TOTAL_REFS()
 #endif /* Py_REF_DEBUG */
 
 #ifdef COUNT_ALLOCS

Include/pystate.h

@@ -53,6 +53,7 @@ typedef struct _is {
 
     int64_t id;
 
+    PyObject *modules;
     PyObject *modules_by_index;
     PyObject *sysdict;
     PyObject *builtins;
@@ -60,6 +61,8 @@ typedef struct _is {
 
     /* Used in Python/sysmodule.c. */
     int check_interval;
+    PyObject *warnoptions;
+    PyObject *xoptions;
 
     /* Used in Modules/_threadmodule.c. */
     long num_threads;

Misc/NEWS.d/next/Core and Builtins/2017-09-04-10-46-09.bpo-28411.IU9rQL.rst

@@ -1,4 +0,0 @@
-``PyInterpreterState`` has a "modules" field that is copied into
-``sys.modules`` during interpreter startup.  This causes problems if a
-program replaces ``sys.modules`` with something else.  To solve this we
-eliminate ``PyInterpreterState.modules``.

Modules/_pickle.c

@@ -6425,7 +6425,9 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
 /*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/
 {
     PyObject *global;
+    PyObject *modules_dict;
     PyObject *module;
+    _Py_IDENTIFIER(modules);
 
     /* Try to map the old names used in Python 2.x to the new ones used in
        Python 3.x.  We do this only with old pickle protocols and when the
@@ -6482,7 +6484,13 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
         }
     }
 
-    module = PyImport_GetModule(module_name);
+    modules_dict = _PySys_GetObjectId(&PyId_modules);
+    if (modules_dict == NULL) {
+        PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
+        return NULL;
+    }
+
+    module = PyDict_GetItemWithError(modules_dict, module_name);
     if (module == NULL) {
         if (PyErr_Occurred())
             return NULL;
@@ -6490,11 +6498,11 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self,
         if (module == NULL)
             return NULL;
         global = getattribute(module, global_name, self->proto >= 4);
+        Py_DECREF(module);
     }
     else {
         global = getattribute(module, global_name, self->proto >= 4);
     }
-    Py_DECREF(module);
     return global;
 }
 

Modules/pyexpat.c

@@ -1643,6 +1643,7 @@ MODULE_INITFUNC(void)
     PyObject *errors_module;
     PyObject *modelmod_name;
     PyObject *model_module;
+    PyObject *sys_modules;
     PyObject *tmpnum, *tmpstr;
     PyObject *codes_dict;
     PyObject *rev_codes_dict;
@@ -1692,6 +1693,11 @@ MODULE_INITFUNC(void)
     */
     PyModule_AddStringConstant(m, "native_encoding", "UTF-8");
 
+    sys_modules = PySys_GetObject("modules");
+    if (sys_modules == NULL) {
+        Py_DECREF(m);
+        return NULL;
+    }
     d = PyModule_GetDict(m);
     if (d == NULL) {
         Py_DECREF(m);
@@ -1701,7 +1707,7 @@ MODULE_INITFUNC(void)
     if (errors_module == NULL) {
         errors_module = PyModule_New(MODULE_NAME ".errors");
         if (errors_module != NULL) {
-            _PyImport_SetModule(errmod_name, errors_module);
+            PyDict_SetItem(sys_modules, errmod_name, errors_module);
             /* gives away the reference to errors_module */
             PyModule_AddObject(m, "errors", errors_module);
         }
@@ -1711,7 +1717,7 @@ MODULE_INITFUNC(void)
     if (model_module == NULL) {
         model_module = PyModule_New(MODULE_NAME ".model");
         if (model_module != NULL) {
-            _PyImport_SetModule(modelmod_name, model_module);
+            PyDict_SetItem(sys_modules, modelmod_name, model_module);
             /* gives away the reference to model_module */
             PyModule_AddObject(m, "model", model_module);
         }

Objects/moduleobject.c

@@ -161,18 +161,12 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions)
 
 PyObject *
 PyModule_Create2(struct PyModuleDef* module, int module_api_version)
-{
-    if (!_PyImport_IsInitialized(PyThreadState_GET()->interp))
-        Py_FatalError("Python import machinery not initialized");
-    return _PyModule_CreateInitialized(module, module_api_version);
-}
-
-PyObject *
-_PyModule_CreateInitialized(struct PyModuleDef* module, int module_api_version)
 {
     const char* name;
     PyModuleObject *m;
-
+    PyInterpreterState *interp = PyThreadState_Get()->interp;
+    if (interp->modules == NULL)
+        Py_FatalError("Python import machinery not initialized");
     if (!PyModuleDef_Init(module))
         return NULL;
     name = module->m_name;

Objects/object.c

@@ -29,23 +29,20 @@ _Py_GetRefTotal(void)
     return total;
 }
 
-PyObject *
-_PyDebug_XOptionShowRefCount(void)
-{
-    PyObject *xoptions = PySys_GetXOptions();
-    if (xoptions == NULL)
-        return NULL;
-
-    _Py_IDENTIFIER(showrefcount);
-    return _PyDict_GetItemId(xoptions, &PyId_showrefcount);
-}
-
 void
 _PyDebug_PrintTotalRefs(void) {
-    fprintf(stderr,
+    PyObject *xoptions, *value;
-            "[%" PY_FORMAT_SIZE_T "d refs, "
+    _Py_IDENTIFIER(showrefcount);
-            "%" PY_FORMAT_SIZE_T "d blocks]\n",
+
-            _Py_GetRefTotal(), _Py_GetAllocatedBlocks());
+    xoptions = PySys_GetXOptions();
+    if (xoptions == NULL)
+        return;
+    value = _PyDict_GetItemId(xoptions, &PyId_showrefcount);
+    if (value == Py_True)
+        fprintf(stderr,
+                "[%" PY_FORMAT_SIZE_T "d refs, "
+                "%" PY_FORMAT_SIZE_T "d blocks]\n",
+                _Py_GetRefTotal(), _Py_GetAllocatedBlocks());
 }
 #endif /* Py_REF_DEBUG */
 

Objects/typeobject.c

@@ -3902,6 +3902,7 @@ import_copyreg(void)
 {
     PyObject *copyreg_str;
     PyObject *copyreg_module;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
     _Py_IDENTIFIER(copyreg);
 
     copyreg_str = _PyUnicode_FromId(&PyId_copyreg);
@@ -3913,7 +3914,7 @@ import_copyreg(void)
        by storing a reference to the cached module in a static variable, but
        this broke when multiple embedded interpreters were in use (see issue
        #17408 and #19088). */
-    copyreg_module = _PyImport_GetModuleWithError(copyreg_str);
+    copyreg_module = PyDict_GetItemWithError(interp->modules, copyreg_str);
     if (copyreg_module != NULL) {
         Py_INCREF(copyreg_module);
         return copyreg_module;

Python/_warnings.c

@@ -38,6 +38,7 @@ static PyObject *
 get_warnings_attr(const char *attr, int try_import)
 {
     static PyObject *warnings_str = NULL;
+    PyObject *all_modules;
     PyObject *warnings_module, *obj;
 
     if (warnings_str == NULL) {
@@ -57,7 +58,9 @@ get_warnings_attr(const char *attr, int try_import)
         }
     }
     else {
-        warnings_module = _PyImport_GetModule(warnings_str);
+        all_modules = PyImport_GetModuleDict();
+
+        warnings_module = PyDict_GetItem(all_modules, warnings_str);
         if (warnings_module == NULL)
             return NULL;
 

Python/bltinmodule.c

@@ -2685,7 +2685,7 @@ _PyBuiltin_Init(void)
         PyType_Ready(&PyZip_Type) < 0)
         return NULL;
 
-    mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION);
+    mod = PyModule_Create(&builtinsmodule);
     if (mod == NULL)
         return NULL;
     dict = PyModule_GetDict(mod);

Python/ceval.c

@@ -4935,7 +4935,7 @@ import_from(PyObject *v, PyObject *name)
         Py_DECREF(pkgname);
         return NULL;
     }
-    x = _PyImport_GetModule(fullmodname);
+    x = PyDict_GetItem(PyImport_GetModuleDict(), fullmodname);
     Py_DECREF(fullmodname);
     if (x == NULL) {
         goto error;

Python/import.c

@@ -290,115 +290,10 @@ _PyImport_Fini(void)
 PyObject *
 PyImport_GetModuleDict(void)
 {
-    PyObject *sysdict = PyThreadState_GET()->interp->sysdict;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
-    if (sysdict == NULL) {
+    if (interp->modules == NULL)
-        Py_FatalError("PyImport_GetModuleDict: no sys module!");
+        Py_FatalError("PyImport_GetModuleDict: no module dictionary!");
-    }
+    return interp->modules;
-
-    _Py_IDENTIFIER(modules);
-    PyObject *modules = _PyDict_GetItemId(sysdict, &PyId_modules);
-    if (modules == NULL) {
-        Py_FatalError("lost sys.modules");
-    }
-    return modules;
-}
-
-/* In some corner cases it is important to be sure that the import
-   machinery has been initialized (or not cleaned up yet).  For
-   example, see issue #4236 and PyModule_Create2(). */
-
-int
-_PyImport_IsInitialized(PyInterpreterState *interp)
-{
-    if (interp->sysdict == NULL)
-        return 0;
-    _Py_IDENTIFIER(modules);
-    PyObject *modules = _PyDict_GetItemId(interp->sysdict, &PyId_modules);
-    if (modules == NULL)
-        return 0;
-    return 1;
-}
-
-PyObject *
-_PyImport_GetModule(PyObject *name)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    if (PyDict_CheckExact(modules)) {
-        return PyDict_GetItem(modules, name);
-    }
-
-    PyObject *mod = PyObject_GetItem(modules, name);
-    // For backward-comaptibility we copy the behavior of PyDict_GetItem().
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
-    }
-    Py_XDECREF(mod);
-    return mod;
-}
-
-PyObject *
-_PyImport_GetModuleWithError(PyObject *name)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    if (PyDict_CheckExact(modules)) {
-        return PyDict_GetItemWithError(modules, name);
-    }
-
-    PyObject *mod = PyObject_GetItem(modules, name);
-    // For backward-comaptibility we copy the behavior
-    // of PyDict_GetItemWithError().
-    if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-        PyErr_Clear();
-    }
-    return mod;
-}
-
-PyObject *
-_PyImport_GetModuleId(struct _Py_Identifier *nameid)
-{
-    PyObject *name = _PyUnicode_FromId(nameid); /* borrowed */
-    if (name == NULL) {
-        return NULL;
-    }
-    return _PyImport_GetModule(name);
-}
-
-int
-_PyImport_SetModule(PyObject *name, PyObject *m)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return PyObject_SetItem(modules, name, m);
-}
-
-int
-_PyImport_SetModuleString(const char *name, PyObject *m)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return PyMapping_SetItemString(modules, name, m);
-}
-
-PyObject *
-PyImport_GetModule(PyObject *name)
-{
-    PyObject *m;
-    PyObject *modules = PyImport_GetModuleDict();
-    if (modules == NULL) {
-        PyErr_SetString(PyExc_RuntimeError, "unable to get sys.modules");
-        return NULL;
-    }
-    Py_INCREF(modules);
-    if (PyDict_CheckExact(modules)) {
-        m = PyDict_GetItemWithError(modules, name);  /* borrowed */
-        Py_XINCREF(m);
-    }
-    else {
-        m = PyObject_GetItem(modules, name);
-        if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-            PyErr_Clear();
-        }
-    }
-    Py_DECREF(modules);
-    return m;
 }
 
 
@@ -428,7 +323,7 @@ PyImport_Cleanup(void)
     Py_ssize_t pos;
     PyObject *key, *value, *dict;
     PyInterpreterState *interp = PyThreadState_GET()->interp;
-    PyObject *modules = PyImport_GetModuleDict();
+    PyObject *modules = interp->modules;
     PyObject *weaklist = NULL;
     const char * const *p;
 
@@ -490,7 +385,7 @@ PyImport_Cleanup(void)
             if (Py_VerboseFlag && PyUnicode_Check(key))
                 PySys_FormatStderr("# cleanup[2] removing %U\n", key);
             STORE_MODULE_WEAKREF(key, value);
-            PyObject_SetItem(modules, key, Py_None);
+            PyDict_SetItem(modules, key, Py_None);
         }
     }
 
@@ -557,6 +452,7 @@ PyImport_Cleanup(void)
     /* Clear and delete the modules directory.  Actual modules will
        still be there only if imported during the execution of some
        destructor. */
+    interp->modules = NULL;
     Py_DECREF(modules);
 
     /* Once more */
@@ -615,9 +511,9 @@ PyImport_GetMagicTag(void)
 
 int
 _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
-                                 PyObject *filename, PyObject *modules)
+                               PyObject *filename)
 {
-    PyObject *dict, *key;
+    PyObject *modules, *dict, *key;
     struct PyModuleDef *def;
     int res;
     if (extensions == NULL) {
@@ -634,10 +530,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
         PyErr_BadInternalCall();
         return -1;
     }
-    if (PyObject_SetItem(modules, name, mod) < 0)
+    modules = PyImport_GetModuleDict();
+    if (PyDict_SetItem(modules, name, mod) < 0)
         return -1;
     if (_PyState_AddModule(mod, def) < 0) {
-        PyMapping_DelItem(modules, name);
+        PyDict_DelItem(modules, name);
         return -1;
     }
     if (def->m_size == -1) {
@@ -665,28 +562,20 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name,
 }
 
 int
-_PyImport_FixupBuiltin(PyObject *mod, const char *name, PyObject *modules)
+_PyImport_FixupBuiltin(PyObject *mod, const char *name)
 {
     int res;
     PyObject *nameobj;
     nameobj = PyUnicode_InternFromString(name);
     if (nameobj == NULL)
         return -1;
-    res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj, modules);
+    res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj);
     Py_DECREF(nameobj);
     return res;
 }
 
 PyObject *
 _PyImport_FindExtensionObject(PyObject *name, PyObject *filename)
-{
-    PyObject *modules = PyImport_GetModuleDict();
-    return _PyImport_FindExtensionObjectEx(name, filename, modules);
-}
-
-PyObject *
-_PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
-                                PyObject *modules)
 {
     PyObject *mod, *mdict, *key;
     PyModuleDef* def;
@@ -703,7 +592,7 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
         /* Module does not support repeated initialization */
         if (def->m_base.m_copy == NULL)
             return NULL;
-        mod = _PyImport_AddModuleObject(name, modules);
+        mod = PyImport_AddModuleObject(name);
         if (mod == NULL)
             return NULL;
         mdict = PyModule_GetDict(mod);
@@ -718,14 +607,14 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
         mod = def->m_base.m_init();
         if (mod == NULL)
             return NULL;
-        if (PyObject_SetItem(modules, name, mod) == -1) {
+        if (PyDict_SetItem(PyImport_GetModuleDict(), name, mod) == -1) {
             Py_DECREF(mod);
             return NULL;
         }
         Py_DECREF(mod);
     }
     if (_PyState_AddModule(mod, def) < 0) {
-        PyMapping_DelItem(modules, name);
+        PyDict_DelItem(PyImport_GetModuleDict(), name);
         Py_DECREF(mod);
         return NULL;
     }
@@ -737,13 +626,13 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename,
 }
 
 PyObject *
-_PyImport_FindBuiltin(const char *name, PyObject *modules)
+_PyImport_FindBuiltin(const char *name)
 {
     PyObject *res, *nameobj;
     nameobj = PyUnicode_InternFromString(name);
     if (nameobj == NULL)
         return NULL;
-    res = _PyImport_FindExtensionObjectEx(nameobj, nameobj, modules);
+    res = _PyImport_FindExtensionObject(nameobj, nameobj);
     Py_DECREF(nameobj);
     return res;
 }
@@ -758,34 +647,19 @@ PyObject *
 PyImport_AddModuleObject(PyObject *name)
 {
     PyObject *modules = PyImport_GetModuleDict();
-    return _PyImport_AddModuleObject(name, modules);
-}
-
-PyObject *
-_PyImport_AddModuleObject(PyObject *name, PyObject *modules)
-{
     PyObject *m;
-    if (PyDict_CheckExact(modules)) {
+
-        m = PyDict_GetItemWithError(modules, name);
+    if ((m = PyDict_GetItemWithError(modules, name)) != NULL &&
-    }
+        PyModule_Check(m)) {
-    else {
+        return m;
-        m = PyObject_GetItem(modules, name);
-        // For backward-comaptibility we copy the behavior
-        // of PyDict_GetItemWithError().
-        if (PyErr_ExceptionMatches(PyExc_KeyError)) {
-            PyErr_Clear();
-        }
     }
     if (PyErr_Occurred()) {
         return NULL;
     }
-    if (m != NULL && PyModule_Check(m)) {
-        return m;
-    }
     m = PyModule_NewObject(name);
     if (m == NULL)
         return NULL;
-    if (PyObject_SetItem(modules, name, m) != 0) {
+    if (PyDict_SetItem(modules, name, m) != 0) {
         Py_DECREF(m);
         return NULL;
     }
@@ -812,13 +686,11 @@ static void
 remove_module(PyObject *name)
 {
     PyObject *modules = PyImport_GetModuleDict();
-    if (PyMapping_DelItem(modules, name) < 0) {
+    if (PyDict_GetItem(modules, name) == NULL)
-        if (!PyMapping_HasKey(modules, name)) {
+        return;
-            return;
+    if (PyDict_DelItem(modules, name) < 0)
-        }
         Py_FatalError("import:  deleting existing key in"
                       "sys.modules failed");
-    }
 }
 
 
@@ -927,6 +799,7 @@ module_dict_for_exec(PyObject *name)
 static PyObject *
 exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object)
 {
+    PyObject *modules = PyImport_GetModuleDict();
     PyObject *v, *m;
 
     v = PyEval_EvalCode(code_object, module_dict, module_dict);
@@ -936,8 +809,7 @@ exec_code_in_module(PyObject *name, PyObject *module_dict, PyObject *code_object
     }
     Py_DECREF(v);
 
-    m = _PyImport_GetModule(name);
+    if ((m = PyDict_GetItem(modules, name)) == NULL) {
-    if (m == NULL) {
         PyErr_Format(PyExc_ImportError,
                      "Loaded module %R not found in sys.modules",
                      name);
@@ -1170,7 +1042,6 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
         return NULL;
     }
 
-    PyObject *modules = NULL;
     for (p = PyImport_Inittab; p->name != NULL; p++) {
         PyModuleDef *def;
         if (_PyUnicode_EqualToASCIIString(name, p->name)) {
@@ -1196,11 +1067,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
                     return NULL;
                 }
                 def->m_base.m_init = p->initfunc;
-                if (modules == NULL) {
+                if (_PyImport_FixupExtensionObject(mod, name, name) < 0) {
-                    modules = PyImport_GetModuleDict();
-                }
-                if (_PyImport_FixupExtensionObject(mod, name, name,
-                                                   modules) < 0) {
                     Py_DECREF(name);
                     return NULL;
                 }
@@ -1644,7 +1511,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
         Py_INCREF(abs_name);
     }
 
-    mod = _PyImport_GetModule(abs_name);
+    mod = PyDict_GetItem(interp->modules, abs_name);
     if (mod != NULL && mod != Py_None) {
         _Py_IDENTIFIER(__spec__);
         _Py_IDENTIFIER(_initializing);
@@ -1731,7 +1598,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals,
                     goto error;
                 }
 
-                final_mod = _PyImport_GetModule(to_return);
+                final_mod = PyDict_GetItem(interp->modules, to_return);
                 Py_DECREF(to_return);
                 if (final_mod == NULL) {
                     PyErr_Format(PyExc_KeyError,
@@ -1784,10 +1651,10 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals
 PyObject *
 PyImport_ReloadModule(PyObject *m)
 {
-    _Py_IDENTIFIER(imp);
     _Py_IDENTIFIER(reload);
     PyObject *reloaded_module = NULL;
-    PyObject *imp = _PyImport_GetModuleId(&PyId_imp);
+    PyObject *modules = PyImport_GetModuleDict();
+    PyObject *imp = PyDict_GetItemString(modules, "imp");
     if (imp == NULL) {
         imp = PyImport_ImportModule("imp");
         if (imp == NULL) {
@@ -1822,6 +1689,7 @@ PyImport_Import(PyObject *module_name)
     PyObject *globals = NULL;
     PyObject *import = NULL;
     PyObject *builtins = NULL;
+    PyObject *modules = NULL;
     PyObject *r = NULL;
 
     /* Initialize constant string objects */
@@ -1876,7 +1744,8 @@ PyImport_Import(PyObject *module_name)
         goto err;
     Py_DECREF(r);
 
-    r = _PyImport_GetModule(module_name);
+    modules = PyImport_GetModuleDict();
+    r = PyDict_GetItemWithError(modules, module_name);
     if (r != NULL) {
         Py_INCREF(r);
     }

Python/importdl.c

@@ -215,8 +215,7 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp)
     else
         Py_INCREF(path);
 
-    PyObject *modules = PyImport_GetModuleDict();
+    if (_PyImport_FixupExtensionObject(m, name_unicode, path) < 0)
-    if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0)
         goto error;
 
     Py_DECREF(name_unicode);

Python/pylifecycle.c

@@ -42,7 +42,6 @@ _Py_IDENTIFIER(name);
 _Py_IDENTIFIER(stdin);
 _Py_IDENTIFIER(stdout);
 _Py_IDENTIFIER(stderr);
-_Py_IDENTIFIER(threading);
 
 #ifdef __cplusplus
 extern "C" {
@@ -284,6 +283,7 @@ initimport(PyInterpreterState *interp, PyObject *sysmod)
 {
     PyObject *importlib;
     PyObject *impmod;
+    PyObject *sys_modules;
     PyObject *value;
 
     /* Import _importlib through its frozen version, _frozen_importlib. */
@@ -314,7 +314,11 @@ initimport(PyInterpreterState *interp, PyObject *sysmod)
     else if (Py_VerboseFlag) {
         PySys_FormatStderr("import _imp # builtin\n");
     }
-    if (_PyImport_SetModuleString("_imp", impmod) < 0) {
+    sys_modules = PyImport_GetModuleDict();
+    if (Py_VerboseFlag) {
+        PySys_FormatStderr("import sys # builtin\n");
+    }
+    if (PyDict_SetItemString(sys_modules, "_imp", impmod) < 0) {
         Py_FatalError("Py_Initialize: can't save _imp to sys.modules");
     }
 
@@ -671,20 +675,10 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     if (!_PyFloat_Init())
         Py_FatalError("Py_InitializeCore: can't init float");
 
-    PyObject *modules = PyDict_New();
+    interp->modules = PyDict_New();
-    if (modules == NULL)
+    if (interp->modules == NULL)
         Py_FatalError("Py_InitializeCore: can't make modules dictionary");
 
-    sysmod = _PySys_BeginInit();
-    if (sysmod == NULL)
-        Py_FatalError("Py_InitializeCore: can't initialize sys");
-    interp->sysdict = PyModule_GetDict(sysmod);
-    if (interp->sysdict == NULL)
-        Py_FatalError("Py_InitializeCore: can't initialize sys dict");
-    Py_INCREF(interp->sysdict);
-    PyDict_SetItemString(interp->sysdict, "modules", modules);
-    _PyImport_FixupBuiltin(sysmod, "sys", modules);
-
     /* Init Unicode implementation; relies on the codec registry */
     if (_PyUnicode_Init() < 0)
         Py_FatalError("Py_InitializeCore: can't initialize unicode");
@@ -695,7 +689,7 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     bimod = _PyBuiltin_Init();
     if (bimod == NULL)
         Py_FatalError("Py_InitializeCore: can't initialize builtins modules");
-    _PyImport_FixupBuiltin(bimod, "builtins", modules);
+    _PyImport_FixupBuiltin(bimod, "builtins");
     interp->builtins = PyModule_GetDict(bimod);
     if (interp->builtins == NULL)
         Py_FatalError("Py_InitializeCore: can't initialize builtins dict");
@@ -704,6 +698,17 @@ void _Py_InitializeCore(const _PyCoreConfig *config)
     /* initialize builtin exceptions */
     _PyExc_Init(bimod);
 
+    sysmod = _PySys_BeginInit();
+    if (sysmod == NULL)
+        Py_FatalError("Py_InitializeCore: can't initialize sys");
+    interp->sysdict = PyModule_GetDict(sysmod);
+    if (interp->sysdict == NULL)
+        Py_FatalError("Py_InitializeCore: can't initialize sys dict");
+    Py_INCREF(interp->sysdict);
+    _PyImport_FixupBuiltin(sysmod, "sys");
+    PyDict_SetItemString(interp->sysdict, "modules",
+                         interp->modules);
+
     /* Set up a preliminary stderr printer until we have enough
        infrastructure for the io module in place. */
     pstderr = PyFile_NewStdPrinter(fileno(stderr));
@@ -1006,11 +1011,6 @@ Py_FinalizeEx(void)
     while (_PyGC_CollectIfEnabled() > 0)
         /* nothing */;
 #endif
-
-#ifdef Py_REF_DEBUG
-    PyObject *showrefcount = _PyDebug_XOptionShowRefCount();
-#endif
-
     /* Destroy all modules */
     PyImport_Cleanup();
 
@@ -1058,10 +1058,7 @@ Py_FinalizeEx(void)
     /* dump hash stats */
     _PyHash_Fini();
 
-#ifdef Py_REF_DEBUG
+    _PY_DEBUG_PRINT_TOTAL_REFS();
-        if (showrefcount == Py_True)
-            _PyDebug_PrintTotalRefs();
-#endif
 
 #ifdef Py_TRACE_REFS
     /* Display all objects still alive -- this can invoke arbitrary
@@ -1206,22 +1203,9 @@ Py_NewInterpreter(void)
 
     /* XXX The following is lax in error checking */
 
-    PyObject *modules = PyDict_New();
+    interp->modules = PyDict_New();
-    if (modules == NULL)
-        Py_FatalError("Py_NewInterpreter: can't make modules dictionary");
 
-    sysmod = _PyImport_FindBuiltin("sys", modules);
+    bimod = _PyImport_FindBuiltin("builtins");
-    if (sysmod != NULL) {
-        interp->sysdict = PyModule_GetDict(sysmod);
-        if (interp->sysdict == NULL)
-            goto handle_error;
-        Py_INCREF(interp->sysdict);
-        PyDict_SetItemString(interp->sysdict, "modules", modules);
-        PySys_SetPath(Py_GetPath());
-        _PySys_EndInit(interp->sysdict);
-    }
-
-    bimod = _PyImport_FindBuiltin("builtins", modules);
     if (bimod != NULL) {
         interp->builtins = PyModule_GetDict(bimod);
         if (interp->builtins == NULL)
@@ -1232,9 +1216,18 @@ Py_NewInterpreter(void)
     /* initialize builtin exceptions */
     _PyExc_Init(bimod);
 
+    sysmod = _PyImport_FindBuiltin("sys");
     if (bimod != NULL && sysmod != NULL) {
         PyObject *pstderr;
 
+        interp->sysdict = PyModule_GetDict(sysmod);
+        if (interp->sysdict == NULL)
+            goto handle_error;
+        Py_INCREF(interp->sysdict);
+        _PySys_EndInit(interp->sysdict);
+        PySys_SetPath(Py_GetPath());
+        PyDict_SetItemString(interp->sysdict, "modules",
+                             interp->modules);
         /* Set up a preliminary stderr printer until we have enough
            infrastructure for the io module in place. */
         pstderr = PyFile_NewStdPrinter(fileno(stderr));
@@ -1910,13 +1903,14 @@ wait_for_thread_shutdown(void)
 {
     _Py_IDENTIFIER(_shutdown);
     PyObject *result;
-    PyObject *threading = _PyImport_GetModuleId(&PyId_threading);
+    PyThreadState *tstate = PyThreadState_GET();
+    PyObject *threading = PyMapping_GetItemString(tstate->interp->modules,
+                                                  "threading");
     if (threading == NULL) {
         /* threading not imported */
         PyErr_Clear();
         return;
     }
-    Py_INCREF(threading);
     result = _PyObject_CallMethodId(threading, &PyId__shutdown, NULL);
     if (result == NULL) {
         PyErr_WriteUnraisable(threading);

Python/pystate.c

@@ -90,12 +90,15 @@ PyInterpreterState_New(void)
                                  PyMem_RawMalloc(sizeof(PyInterpreterState));
 
     if (interp != NULL) {
+        interp->modules = NULL;
         interp->modules_by_index = NULL;
         interp->sysdict = NULL;
         interp->builtins = NULL;
         interp->builtins_copy = NULL;
         interp->tstate_head = NULL;
         interp->check_interval = 100;
+        interp->warnoptions = NULL;
+        interp->xoptions = NULL;
         interp->num_threads = 0;
         interp->pythread_stacksize = 0;
         interp->codec_search_path = NULL;
@@ -153,6 +156,7 @@ PyInterpreterState_Clear(PyInterpreterState *interp)
     Py_CLEAR(interp->codec_search_path);
     Py_CLEAR(interp->codec_search_cache);
     Py_CLEAR(interp->codec_error_registry);
+    Py_CLEAR(interp->modules);
     Py_CLEAR(interp->modules_by_index);
     Py_CLEAR(interp->sysdict);
     Py_CLEAR(interp->builtins);

Python/pythonrun.c

@@ -113,10 +113,7 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename_str, PyCompilerFlags *
     err = -1;
     for (;;) {
         ret = PyRun_InteractiveOneObject(fp, filename, flags);
-#ifdef Py_REF_DEBUG
+        _PY_DEBUG_PRINT_TOTAL_REFS();
-        if (_PyDebug_XOptionShowRefCount() == Py_True)
-            _PyDebug_PrintTotalRefs();
-#endif
         if (ret == E_EOF) {
             err = 0;
             break;

Python/sysmodule.c

@@ -36,14 +36,12 @@ extern const char *PyWin_DLLVersionString;
 
 _Py_IDENTIFIER(_);
 _Py_IDENTIFIER(__sizeof__);
-_Py_IDENTIFIER(_xoptions);
 _Py_IDENTIFIER(buffer);
 _Py_IDENTIFIER(builtins);
 _Py_IDENTIFIER(encoding);
 _Py_IDENTIFIER(path);
 _Py_IDENTIFIER(stdout);
 _Py_IDENTIFIER(stderr);
-_Py_IDENTIFIER(warnoptions);
 _Py_IDENTIFIER(write);
 
 PyObject *
@@ -162,11 +160,13 @@ static PyObject *
 sys_displayhook(PyObject *self, PyObject *o)
 {
     PyObject *outf;
+    PyInterpreterState *interp = PyThreadState_GET()->interp;
+    PyObject *modules = interp->modules;
     PyObject *builtins;
     static PyObject *newline = NULL;
     int err;
 
-    builtins = _PyImport_GetModuleId(&PyId_builtins);
+    builtins = _PyDict_GetItemId(modules, &PyId_builtins);
     if (builtins == NULL) {
         PyErr_SetString(PyExc_RuntimeError, "lost builtins module");
         return NULL;
@@ -1481,17 +1481,13 @@ list_builtin_module_names(void)
 static PyObject *
 get_warnoptions(void)
 {
-    PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions);
+    PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions;
     if (warnoptions == NULL || !PyList_Check(warnoptions)) {
         Py_XDECREF(warnoptions);
         warnoptions = PyList_New(0);
         if (warnoptions == NULL)
             return NULL;
-        if (_PySys_SetObjectId(&PyId_warnoptions, warnoptions)) {
+        PyThreadState_GET()->interp->warnoptions = warnoptions;
-            Py_DECREF(warnoptions);
-            return NULL;
-        }
-        Py_DECREF(warnoptions);
     }
     return warnoptions;
 }
@@ -1499,7 +1495,7 @@ get_warnoptions(void)
 void
 PySys_ResetWarnOptions(void)
 {
-    PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions);
+    PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions;
     if (warnoptions == NULL || !PyList_Check(warnoptions))
         return;
     PyList_SetSlice(warnoptions, 0, PyList_GET_SIZE(warnoptions), NULL);
@@ -1528,24 +1524,20 @@ PySys_AddWarnOption(const wchar_t *s)
 int
 PySys_HasWarnOptions(void)
 {
-    PyObject *warnoptions = _PySys_GetObjectId(&PyId_warnoptions);
+    PyObject *warnoptions = PyThreadState_GET()->interp->warnoptions;
     return (warnoptions != NULL && (PyList_Size(warnoptions) > 0)) ? 1 : 0;
 }
 
 static PyObject *
 get_xoptions(void)
 {
-    PyObject *xoptions = _PySys_GetObjectId(&PyId__xoptions);
+    PyObject *xoptions = PyThreadState_GET()->interp->xoptions;
     if (xoptions == NULL || !PyDict_Check(xoptions)) {
         Py_XDECREF(xoptions);
         xoptions = PyDict_New();
         if (xoptions == NULL)
             return NULL;
-        if (_PySys_SetObjectId(&PyId__xoptions, xoptions)) {
+        PyThreadState_GET()->interp->xoptions = xoptions;
-            Py_DECREF(xoptions);
-            return NULL;
-        }
-        Py_DECREF(xoptions);
     }
     return xoptions;
 }
@@ -1947,7 +1939,7 @@ _PySys_BeginInit(void)
     PyObject *m, *sysdict, *version_info;
     int res;
 
-    m = _PyModule_CreateInitialized(&sysmodule, PYTHON_API_VERSION);
+    m = PyModule_Create(&sysmodule);
     if (m == NULL)
         return NULL;
     sysdict = PyModule_GetDict(m);
@@ -2094,6 +2086,16 @@ _PySys_BeginInit(void)
 #undef SET_SYS_FROM_STRING_BORROW
 
 /* Updating the sys namespace, returning integer error codes */
+#define SET_SYS_FROM_STRING_BORROW_INT_RESULT(key, value)  \
+    do {                                                   \
+        PyObject *v = (value);                             \
+        if (v == NULL)                                     \
+            return -1;                                     \
+        res = PyDict_SetItemString(sysdict, key, v);       \
+        if (res < 0) {                                     \
+            return res;                                    \
+        }                                                  \
+    } while (0)
 #define SET_SYS_FROM_STRING_INT_RESULT(key, value)         \
     do {                                                   \
         PyObject *v = (value);                             \
@@ -2138,11 +2140,15 @@ _PySys_EndInit(PyObject *sysdict)
     SET_SYS_FROM_STRING_INT_RESULT("base_exec_prefix",
                         PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
 
-    if (get_warnoptions() == NULL)
+    PyObject *warnoptions = get_warnoptions();
+    if (warnoptions == NULL)
         return -1;
+    SET_SYS_FROM_STRING_BORROW_INT_RESULT("warnoptions", warnoptions);
 
-    if (get_xoptions() == NULL)
+    PyObject *xoptions = get_xoptions();
+    if (xoptions == NULL)
         return -1;
+    SET_SYS_FROM_STRING_BORROW_INT_RESULT("_xoptions", xoptions);
 
     if (PyErr_Occurred())
         return -1;
@@ -2150,6 +2156,7 @@ _PySys_EndInit(PyObject *sysdict)
 }
 
 #undef SET_SYS_FROM_STRING_INT_RESULT
+#undef SET_SYS_FROM_STRING_BORROW_INT_RESULT
 
 static PyObject *
 makepathobject(const wchar_t *path, wchar_t delim)