bpo-22257: Private C-API for core runtime initialization (PEP 432). (#1772)

(patch by Nick Coghlan)

Include/pylifecycle.h

@@ -19,8 +19,14 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
  */
 PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
                                              const char *errors);
+
+/* PEP 432 Multi-phase initialization API (Private while provisional!) */
+PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *);
+PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
+PyAPI_FUNC(int) _Py_InitializeMainInterpreter(int install_sigs);
 #endif
 
+/* Initialization and finalization */
 PyAPI_FUNC(void) Py_Initialize(void);
 PyAPI_FUNC(void) Py_InitializeEx(int);
 #ifndef Py_LIMITED_API
@@ -29,6 +35,8 @@ PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
 PyAPI_FUNC(void) Py_Finalize(void);
 PyAPI_FUNC(int) Py_FinalizeEx(void);
 PyAPI_FUNC(int) Py_IsInitialized(void);
+
+/* Subinterpreter support */
 PyAPI_FUNC(PyThreadState *) Py_NewInterpreter(void);
 PyAPI_FUNC(void) Py_EndInterpreter(PyThreadState *);
 
@@ -85,7 +93,7 @@ PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(int) _PyFloat_Init(void);
 PyAPI_FUNC(int) PyByteArray_Init(void);
-PyAPI_FUNC(void) _Py_HashRandomization_Init(void);
+PyAPI_FUNC(void) _Py_HashRandomization_Init(_PyCoreConfig *core_config);
 #endif
 
 /* Various internal finalizers */

Include/pystate.h

@@ -23,6 +23,16 @@ typedef struct _is PyInterpreterState;
 #else
 typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
 
+
+typedef struct {
+    int ignore_environment;
+    int use_hash_seed;
+    unsigned long hash_seed;
+    int _disable_importlib; /* Needed by freeze_importlib */
+} _PyCoreConfig;
+
+#define _PyCoreConfig_INIT {0, -1, 0, 0}
+
 typedef struct _is {
 
     struct _is *next;
@@ -42,6 +52,7 @@ typedef struct _is {
     int codecs_initialized;
     int fscodec_initialized;
 
+    _PyCoreConfig core_config;
 #ifdef HAVE_DLOPEN
     int dlopenflags;
 #endif

Modules/main.c

@@ -380,19 +380,6 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline)
     wchar_t *command = NULL;
     wchar_t *module = NULL;
     int c;
-    char *opt;
-
-    opt = Py_GETENV("PYTHONMALLOC");
-    if (_PyMem_SetupAllocators(opt) < 0) {
-        fprintf(stderr,
-                "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt);
-        exit(1);
-    }
-
-	// TODO: Move these to core runtime init.
-    Py_HashRandomizationFlag = 1;
-    _Py_HashRandomization_Init();
-    PySys_ResetWarnOptions();
 
     _PyOS_ResetGetOpt();
 
@@ -584,6 +571,7 @@ Py_Main(int argc, wchar_t **argv)
 #endif
     int stdin_is_interactive = 0;
     _Py_CommandLineDetails cmdline = _Py_CommandLineDetails_INIT;
+    _PyCoreConfig core_config = _PyCoreConfig_INIT;
     PyCompilerFlags cf;
     PyObject *main_importer_path = NULL;
 
@@ -602,11 +590,23 @@ Py_Main(int argc, wchar_t **argv)
             break;
         }
         if (c == 'E' || c == 'I') {
-            Py_IgnoreEnvironmentFlag++;
+            core_config.ignore_environment++;
             break;
         }
     }
 
+    char *pymalloc = Py_GETENV("PYTHONMALLOC");
+    if (_PyMem_SetupAllocators(pymalloc) < 0) {
+        fprintf(stderr,
+            "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", pymalloc);
+        exit(1);
+    }
+
+    /* Initialize the core language runtime */
+    Py_IgnoreEnvironmentFlag = core_config.ignore_environment;
+    core_config._disable_importlib = 0;
+    _Py_InitializeCore(&core_config);
+
     /* Reprocess the command line with the language runtime available */
     if (read_command_line(argc, argv, &cmdline)) {
         return usage(2, argv[0]);
@@ -680,6 +680,7 @@ Py_Main(int argc, wchar_t **argv)
         for (i = 0; i < PyList_GET_SIZE(cmdline.warning_options); i++) {
             PySys_AddWarnOptionUnicode(PyList_GET_ITEM(cmdline.warning_options, i));
         }
+        Py_DECREF(cmdline.warning_options);
     }
 
     stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0);
@@ -767,9 +768,10 @@ Py_Main(int argc, wchar_t **argv)
 #else
     Py_SetProgramName(argv[0]);
 #endif
-    Py_Initialize();
+    if (_Py_InitializeMainInterpreter(1))
-    Py_XDECREF(cmdline.warning_options);
+        Py_FatalError("Py_Main: Py_InitializeMainInterpreter failed");
 
+    /* TODO: Move this to _PyRun_PrepareMain */
     if (!Py_QuietFlag && (Py_VerboseFlag ||
                         (cmdline.command == NULL && cmdline.filename == NULL &&
                          cmdline.module == NULL && stdin_is_interactive))) {
@@ -779,6 +781,7 @@ Py_Main(int argc, wchar_t **argv)
             fprintf(stderr, "%s\n", COPYRIGHT);
     }
 
+    /* TODO: Move this to _Py_InitializeMainInterpreter */
     if (cmdline.command != NULL) {
         /* Backup _PyOS_optind and force sys.argv[0] = '-c' */
         _PyOS_optind--;

Python/bootstrap_hash.c

@@ -599,11 +599,11 @@ init_hash_secret(int use_hash_seed,
 }
 
 void
-_Py_HashRandomization_Init(void)
+_Py_HashRandomization_Init(_PyCoreConfig *core_config)
 {
     char *seed_text;
-    int use_hash_seed = -1;
+    int use_hash_seed = core_config->use_hash_seed;
-    unsigned long hash_seed;
+    unsigned long hash_seed = core_config->hash_seed;
 
     if (use_hash_seed < 0) {
         seed_text = Py_GETENV("PYTHONHASHSEED");
@@ -611,6 +611,8 @@ _Py_HashRandomization_Init(void)
             Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer "
                           "in range [0; 4294967295]");
         }
+        core_config->use_hash_seed = use_hash_seed;
+        core_config->hash_seed = hash_seed;
     }
     init_hash_secret(use_hash_seed, hash_seed);
 }

Python/pylifecycle.c

@@ -112,14 +112,23 @@ PyModule_GetWarningsModule(void)
     return PyImport_ImportModule("warnings");
 }
 
-static int initialized = 0;
+/* APIs to access the initialization flags
+ *
+ * Can be called prior to Py_Initialize.
+ */
+int _Py_CoreInitialized = 0;
+int _Py_Initialized = 0;
 
-/* API to access the initialized flag -- useful for esoteric use */
+int
+_Py_IsCoreInitialized(void)
+{
+    return _Py_CoreInitialized;
+}
 
 int
 Py_IsInitialized(void)
 {
-    return initialized;
+    return _Py_Initialized;
 }
 
 /* Helper to allow an embedding application to override the normal
@@ -246,7 +255,7 @@ get_locale_encoding(void)
 }
 
 static void
-import_init(PyInterpreterState *interp, PyObject *sysmod)
+initimport(PyInterpreterState *interp, PyObject *sysmod)
 {
     PyObject *importlib;
     PyObject *impmod;
@@ -304,19 +313,82 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
     _PyImportZip_Init();
 }
 
+static void
+initexternalimport(PyInterpreterState *interp)
+{
+    PyObject *value;
+    value = PyObject_CallMethod(interp->importlib,
+                                "_install_external_importers", "");
+    if (value == NULL) {
+        PyErr_Print();
+        Py_FatalError("Py_EndInitialization: external importer setup failed");
+    }
+}
 
-void
+
-_Py_InitializeEx_Private(int install_sigs, int install_importlib)
+/* Global initializations.  Can be undone by Py_Finalize().  Don't
+   call this twice without an intervening Py_Finalize() call.
+
+   Every call to Py_InitializeCore, Py_Initialize or Py_InitializeEx
+   must have a corresponding call to Py_Finalize.
+
+   Locking: you must hold the interpreter lock while calling these APIs.
+   (If the lock has not yet been initialized, that's equivalent to
+   having the lock, but you cannot use multiple threads.)
+
+*/
+
+/* Begin interpreter initialization
+ *
+ * On return, the first thread and interpreter state have been created,
+ * but the compiler, signal handling, multithreading and
+ * multiple interpreter support, and codec infrastructure are not yet
+ * available.
+ *
+ * The import system will support builtin and frozen modules only.
+ * The only supported io is writing to sys.stderr
+ *
+ * If any operation invoked by this function fails, a fatal error is
+ * issued and the function does not return.
+ *
+ * Any code invoked from this function should *not* assume it has access
+ * to the Python C API (unless the API is explicitly listed as being
+ * safe to call without calling Py_Initialize first)
+ */
+
+/* TODO: Progresively move functionality from Py_BeginInitialization to
+ * Py_ReadConfig and Py_EndInitialization
+ */
+
+void _Py_InitializeCore(const _PyCoreConfig *config)
 {
     PyInterpreterState *interp;
     PyThreadState *tstate;
     PyObject *bimod, *sysmod, *pstderr;
     char *p;
     extern void _Py_ReadyTypes(void);
+    _PyCoreConfig core_config = _PyCoreConfig_INIT;
 
-    if (initialized)
+    if (config != NULL) {
-        return;
+        core_config = *config;
-    initialized = 1;
+    }
+
+    if (_Py_Initialized) {
+        Py_FatalError("Py_InitializeCore: main interpreter already initialized");
+    }
+    if (_Py_CoreInitialized) {
+        Py_FatalError("Py_InitializeCore: runtime core already initialized");
+    }
+
+    /* Py_Finalize leaves _Py_Finalizing set in order to help daemon
+     * threads behave a little more gracefully at interpreter shutdown.
+     * We clobber it here so the new interpreter can start with a clean
+     * slate.
+     *
+     * However, this may still lead to misbehaviour if there are daemon
+     * threads still hanging around from a previous Py_Initialize/Finalize
+     * pair :(
+     */
     _Py_Finalizing = NULL;
 
 #ifdef HAVE_SETLOCALE
@@ -345,16 +417,21 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
         Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p);
 #endif
 
-    _Py_HashRandomization_Init();
+    _Py_HashRandomization_Init(&core_config);
+    if (!core_config.use_hash_seed || core_config.hash_seed) {
+        /* Random or non-zero hash seed */
+        Py_HashRandomizationFlag = 1;
+    }
 
     _PyInterpreterState_Init();
     interp = PyInterpreterState_New();
     if (interp == NULL)
-        Py_FatalError("Py_Initialize: can't make first interpreter");
+        Py_FatalError("Py_InitializeCore: can't make main interpreter");
+    interp->core_config = core_config;
 
     tstate = PyThreadState_New(interp);
     if (tstate == NULL)
-        Py_FatalError("Py_Initialize: can't make first thread");
+        Py_FatalError("Py_InitializeCore: can't make first thread");
     (void) PyThreadState_Swap(tstate);
 
 #ifdef WITH_THREAD
@@ -364,7 +441,6 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
        Instead we destroy the previously created GIL here, which ensures
        that we can call Py_Initialize / Py_FinalizeEx multiple times. */
     _PyEval_FiniThreads();
-
     /* Auto-thread-state API */
     _PyGILState_Init(interp, tstate);
 #endif /* WITH_THREAD */
@@ -372,34 +448,35 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     _Py_ReadyTypes();
 
     if (!_PyFrame_Init())
-        Py_FatalError("Py_Initialize: can't init frames");
+        Py_FatalError("Py_InitializeCore: can't init frames");
 
     if (!_PyLong_Init())
-        Py_FatalError("Py_Initialize: can't init longs");
+        Py_FatalError("Py_InitializeCore: can't init longs");
 
     if (!PyByteArray_Init())
-        Py_FatalError("Py_Initialize: can't init bytearray");
+        Py_FatalError("Py_InitializeCore: can't init bytearray");
 
     if (!_PyFloat_Init())
-        Py_FatalError("Py_Initialize: can't init float");
+        Py_FatalError("Py_InitializeCore: can't init float");
 
     interp->modules = PyDict_New();
     if (interp->modules == NULL)
-        Py_FatalError("Py_Initialize: can't make modules dictionary");
+        Py_FatalError("Py_InitializeCore: can't make modules dictionary");
 
     /* Init Unicode implementation; relies on the codec registry */
     if (_PyUnicode_Init() < 0)
-        Py_FatalError("Py_Initialize: can't initialize unicode");
+        Py_FatalError("Py_InitializeCore: can't initialize unicode");
+
     if (_PyStructSequence_Init() < 0)
-        Py_FatalError("Py_Initialize: can't initialize structseq");
+        Py_FatalError("Py_InitializeCore: can't initialize structseq");
 
     bimod = _PyBuiltin_Init();
     if (bimod == NULL)
-        Py_FatalError("Py_Initialize: can't initialize builtins modules");
+        Py_FatalError("Py_InitializeCore: can't initialize builtins modules");
     _PyImport_FixupBuiltin(bimod, "builtins");
     interp->builtins = PyModule_GetDict(bimod);
     if (interp->builtins == NULL)
-        Py_FatalError("Py_Initialize: can't initialize builtins dict");
+        Py_FatalError("Py_InitializeCore: can't initialize builtins dict");
     Py_INCREF(interp->builtins);
 
     /* initialize builtin exceptions */
@@ -407,19 +484,11 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
 
     sysmod = _PySys_BeginInit();
     if (sysmod == NULL)
-        Py_FatalError("Py_Initialize: can't initialize sys");
+        Py_FatalError("Py_InitializeCore: can't initialize sys");
     interp->sysdict = PyModule_GetDict(sysmod);
     if (interp->sysdict == NULL)
-        Py_FatalError("Py_Initialize: can't initialize sys dict");
+        Py_FatalError("Py_InitializeCore: can't initialize sys dict");
     Py_INCREF(interp->sysdict);
-
-    /* GetPath may initialize state that _PySys_EndInit locks
-       in, and so has to be called first.
-       
-       Hopefully one day Eric Snow will fix this. */
-    PySys_SetPath(Py_GetPath());
-    if (_PySys_EndInit(interp->sysdict) < 0)
-        Py_FatalError("Py_Initialize: can't initialize sys");
     _PyImport_FixupBuiltin(sysmod, "sys");
     PyDict_SetItemString(interp->sysdict, "modules",
                          interp->modules);
@@ -428,7 +497,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
        infrastructure for the io module in place. */
     pstderr = PyFile_NewStdPrinter(fileno(stderr));
     if (pstderr == NULL)
-        Py_FatalError("Py_Initialize: can't set preliminary stderr");
+        Py_FatalError("Py_InitializeCore: can't set preliminary stderr");
     _PySys_SetObjectId(&PyId_stderr, pstderr);
     PySys_SetObject("__stderr__", pstderr);
     Py_DECREF(pstderr);
@@ -440,13 +509,53 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     /* Initialize _warnings. */
     _PyWarnings_Init();
 
-    if (!install_importlib)
+    /* This call sets up builtin and frozen import support */
-        return;
+    if (!interp->core_config._disable_importlib) {
+        initimport(interp, sysmod);
+    }
+
+    /* Only when we get here is the runtime core fully initialized */
+    _Py_CoreInitialized = 1;
+}
+
+int
+_Py_InitializeMainInterpreter(int install_sigs)
+{
+    PyInterpreterState *interp;
+    PyThreadState *tstate;
+
+    /* Get current thread state and interpreter pointer */
+    tstate = PyThreadState_GET();
+    if (!tstate)
+        Py_FatalError("Py_Initialize: failed to read thread state");
+    interp = tstate->interp;
+    if (!interp)
+        Py_FatalError("Py_Initialize: failed to get interpreter");
+
+    /* Now finish configuring the main interpreter */
+    if (interp->core_config._disable_importlib) {
+        /* Special mode for freeze_importlib: run with no import system
+         *
+         * This means anything which needs support from extension modules
+         * or pure Python code in the standard library won't work.
+         */
+        _Py_Initialized = 1;
+        return 0;
+    }
+    /* TODO: Report exceptions rather than fatal errors below here */
 
     if (_PyTime_Init() < 0)
         Py_FatalError("Py_Initialize: can't initialize time");
 
-    import_init(interp, sysmod);
+    /* Finish setting up the sys module and import system */
+    /* GetPath may initialize state that _PySys_EndInit locks
+       in, and so has to be called first. */
+    PySys_SetPath(Py_GetPath());
+    if (_PySys_EndInit(interp->sysdict) < 0)
+        Py_FatalError("Py_InitializeMainInterpreter: can't finish initializing sys");
+    /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
+    PySys_SetPath(Py_GetPath());
+    initexternalimport(interp);
 
     /* initialize the faulthandler module */
     if (_PyFaulthandler_Init())
@@ -476,10 +585,27 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
         Py_XDECREF(warnings_module);
     }
 
+    _Py_Initialized = 1;
+
     if (!Py_NoSiteFlag)
         initsite(); /* Module site */
+
+	return 0;
 }
 
+void
+_Py_InitializeEx_Private(int install_sigs, int install_importlib)
+{
+    _PyCoreConfig core_config = _PyCoreConfig_INIT;
+
+    /* TODO: Moar config options! */
+    core_config.ignore_environment = Py_IgnoreEnvironmentFlag;
+    core_config._disable_importlib = !install_importlib;
+    _Py_InitializeCore(&core_config);
+    _Py_InitializeMainInterpreter(install_sigs);
+}
+
+
 void
 Py_InitializeEx(int install_sigs)
 {
@@ -567,7 +693,7 @@ Py_FinalizeEx(void)
     PyThreadState *tstate;
     int status = 0;
 
-    if (!initialized)
+    if (!_Py_Initialized)
         return status;
 
     wait_for_thread_shutdown();
@@ -590,7 +716,8 @@ Py_FinalizeEx(void)
     /* Remaining threads (e.g. daemon threads) will automatically exit
        after taking the GIL (in PyEval_RestoreThread()). */
     _Py_Finalizing = tstate;
-    initialized = 0;
+    _Py_Initialized = 0;
+    _Py_CoreInitialized = 0;
 
     /* Flush sys.stdout and sys.stderr */
     if (flush_std_files() < 0) {
@@ -781,7 +908,7 @@ Py_NewInterpreter(void)
     PyThreadState *tstate, *save_tstate;
     PyObject *bimod, *sysmod;
 
-    if (!initialized)
+    if (!_Py_Initialized)
         Py_FatalError("Py_NewInterpreter: call Py_Initialize first");
 
 #ifdef WITH_THREAD
@@ -802,6 +929,15 @@ Py_NewInterpreter(void)
 
     save_tstate = PyThreadState_Swap(tstate);
 
+    /* Copy the current interpreter config into the new interpreter */
+    if (save_tstate != NULL) {
+        interp->core_config = save_tstate->interp->core_config;
+    } else {
+        /* No current thread state, copy from the main interpreter */
+        PyInterpreterState *main_interp = PyInterpreterState_Main();
+        interp->core_config = main_interp->core_config;
+    }
+
     /* XXX The following is lax in error checking */
 
     interp->modules = PyDict_New();
@@ -825,6 +961,7 @@ Py_NewInterpreter(void)
         if (interp->sysdict == NULL)
             goto handle_error;
         Py_INCREF(interp->sysdict);
+        _PySys_EndInit(interp->sysdict);
         PySys_SetPath(Py_GetPath());
         PyDict_SetItemString(interp->sysdict, "modules",
                              interp->modules);
@@ -832,21 +969,22 @@ Py_NewInterpreter(void)
            infrastructure for the io module in place. */
         pstderr = PyFile_NewStdPrinter(fileno(stderr));
         if (pstderr == NULL)
-            Py_FatalError("Py_Initialize: can't set preliminary stderr");
+            Py_FatalError("Py_NewInterpreter: can't set preliminary stderr");
         _PySys_SetObjectId(&PyId_stderr, pstderr);
         PySys_SetObject("__stderr__", pstderr);
         Py_DECREF(pstderr);
 
         _PyImportHooks_Init();
 
-        import_init(interp, sysmod);
+        initimport(interp, sysmod);
+        initexternalimport(interp);
 
         if (initfsencoding(interp) < 0)
             goto handle_error;
 
         if (initstdio() < 0)
             Py_FatalError(
-                "Py_Initialize: can't initialize sys standard streams");
+                "Py_NewInterpreter: can't initialize sys standard streams");
         initmain(interp);
         if (!Py_NoSiteFlag)
             initsite();