bpo-22257: Small changes for PEP 432. (#1728)

PEP 432 specifies a number of large changes to interpreter startup code, including exposing a cleaner C-API. The major changes depend on a number of smaller changes. This patch includes all those smaller changes.

Include/Python.h

@@ -112,6 +112,7 @@
 
 #include "pyarena.h"
 #include "modsupport.h"
+#include "compile.h"
 #include "pythonrun.h"
 #include "pylifecycle.h"
 #include "ceval.h"
@@ -123,7 +124,6 @@
 #include "abstract.h"
 #include "bltinmodule.h"
 
-#include "compile.h"
 #include "eval.h"
 
 #include "pyctype.h"

Include/compile.h

@@ -11,6 +11,23 @@ extern "C" {
 /* Public interface */
 struct _node; /* Declare the existence of this type */
 PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+/* XXX (ncoghlan): Unprefixed type name in a public API! */
+
+#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
+                   CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
+                   CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
+                   CO_FUTURE_GENERATOR_STOP)
+#define PyCF_MASK_OBSOLETE (CO_NESTED)
+#define PyCF_SOURCE_IS_UTF8  0x0100
+#define PyCF_DONT_IMPLY_DEDENT 0x0200
+#define PyCF_ONLY_AST 0x0400
+#define PyCF_IGNORE_COOKIE 0x0800
+
+#ifndef Py_LIMITED_API
+typedef struct {
+    int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
+} PyCompilerFlags;
+#endif
 
 /* Future feature support */
 

Include/pylifecycle.h

@@ -77,14 +77,15 @@ PyAPI_FUNC(const char *) _Py_gitversion(void);
 /* Internal -- various one-time initializations */
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
-PyAPI_FUNC(PyObject *) _PySys_Init(void);
+PyAPI_FUNC(PyObject *) _PySys_BeginInit(void);
+PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict);
 PyAPI_FUNC(void) _PyImport_Init(void);
 PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
 PyAPI_FUNC(void) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(int) _PyFloat_Init(void);
 PyAPI_FUNC(int) PyByteArray_Init(void);
-PyAPI_FUNC(void) _PyRandom_Init(void);
+PyAPI_FUNC(void) _Py_HashRandomization_Init(void);
 #endif
 
 /* Various internal finalizers */
@@ -106,7 +107,7 @@ PyAPI_FUNC(void) _PyGC_DumpShutdownStats(void);
 PyAPI_FUNC(void) _PyGC_Fini(void);
 PyAPI_FUNC(void) PySlice_Fini(void);
 PyAPI_FUNC(void) _PyType_Fini(void);
-PyAPI_FUNC(void) _PyRandom_Fini(void);
+PyAPI_FUNC(void) _Py_HashRandomization_Fini(void);
 PyAPI_FUNC(void) PyAsyncGen_Fini(void);
 
 PyAPI_DATA(PyThreadState *) _Py_Finalizing;

Include/pystate.h

@@ -302,6 +302,7 @@ PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void);
 /* Routines for advanced debuggers, requested by David Beazley.
    Don't use unless you know what you are doing! */
 #ifndef Py_LIMITED_API
+PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void);
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Head(void);
 PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Next(PyInterpreterState *);
 PyAPI_FUNC(PyThreadState *) PyInterpreterState_ThreadHead(PyInterpreterState *);

Include/pythonrun.h

@@ -7,22 +7,6 @@
 extern "C" {
 #endif
 
-#define PyCF_MASK (CO_FUTURE_DIVISION | CO_FUTURE_ABSOLUTE_IMPORT | \
-                   CO_FUTURE_WITH_STATEMENT | CO_FUTURE_PRINT_FUNCTION | \
-                   CO_FUTURE_UNICODE_LITERALS | CO_FUTURE_BARRY_AS_BDFL | \
-                   CO_FUTURE_GENERATOR_STOP)
-#define PyCF_MASK_OBSOLETE (CO_NESTED)
-#define PyCF_SOURCE_IS_UTF8  0x0100
-#define PyCF_DONT_IMPLY_DEDENT 0x0200
-#define PyCF_ONLY_AST 0x0400
-#define PyCF_IGNORE_COOKIE 0x0800
-
-#ifndef Py_LIMITED_API
-typedef struct {
-    int cf_flags;  /* bitmask of CO_xxx flags relevant to future */
-} PyCompilerFlags;
-#endif
-
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
 PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);

Lib/importlib/_bootstrap.py

@@ -1134,12 +1134,15 @@ def _setup(sys_module, _imp_module):
 
 
 def _install(sys_module, _imp_module):
-    """Install importlib as the implementation of import."""
+    """Install importers for builtin and frozen modules"""
     _setup(sys_module, _imp_module)
 
     sys.meta_path.append(BuiltinImporter)
     sys.meta_path.append(FrozenImporter)
 
+
+def _install_external_importers():
+    """Install importers that require external filesystem access"""
     global _bootstrap_external
     import _frozen_importlib_external
     _bootstrap_external = _frozen_importlib_external

Lib/test/coding20731.py

@@ -1,4 +1,4 @@
-#coding:latin1
+#coding:latin1
-
+
-
+
-
+

Lib/test/test_cmd_line.py

@@ -485,8 +485,29 @@ class CmdLineTest(unittest.TestCase):
                                           cwd=tmpdir)
             self.assertEqual(out.strip(), b"ok")
 
+
+class IgnoreEnvironmentTest(unittest.TestCase):
+
+    def run_ignoring_vars(self, predicate, **env_vars):
+        # Runs a subprocess with -E set, even though we're passing
+        # specific environment variables
+        # Logical inversion to match predicate check to a zero return
+        # code indicating success
+        code = "import sys; sys.stderr.write(str(sys.flags)); sys.exit(not ({}))".format(predicate)
+        return assert_python_ok('-E', '-c', code, **env_vars)
+
+    def test_ignore_PYTHONPATH(self):
+        path = "should_be_ignored"
+        self.run_ignoring_vars("'{}' not in sys.path".format(path),
+                               PYTHONPATH=path)
+
+    def test_ignore_PYTHONHASHSEED(self):
+        self.run_ignoring_vars("sys.flags.hash_randomization == 1",
+                               PYTHONHASHSEED="0")
+
+
 def test_main():
-    test.support.run_unittest(CmdLineTest)
+    test.support.run_unittest(CmdLineTest, IgnoreEnvironmentTest)
     test.support.reap_children()
 
 if __name__ == "__main__":

Lib/test/test_site.py

@@ -183,6 +183,7 @@ class HelperFunctionsTests(unittest.TestCase):
     @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 "
                           "user-site (site.ENABLE_USER_SITE)")
     def test_s_option(self):
+        # (ncoghlan) Change this to use script_helper...
         usersite = site.USER_SITE
         self.assertIn(usersite, sys.path)
 
@@ -199,7 +200,7 @@ class HelperFunctionsTests(unittest.TestCase):
         if usersite == site.getsitepackages()[0]:
             self.assertEqual(rc, 1)
         else:
-            self.assertEqual(rc, 0)
+            self.assertEqual(rc, 0, "User site still added to path with -s")
 
         env = os.environ.copy()
         env["PYTHONNOUSERSITE"] = "1"
@@ -209,14 +210,16 @@ class HelperFunctionsTests(unittest.TestCase):
         if usersite == site.getsitepackages()[0]:
             self.assertEqual(rc, 1)
         else:
-            self.assertEqual(rc, 0)
+            self.assertEqual(rc, 0,
+                        "User site still added to path with PYTHONNOUSERSITE")
 
         env = os.environ.copy()
         env["PYTHONUSERBASE"] = "/tmp"
         rc = subprocess.call([sys.executable, '-c',
             'import sys, site; sys.exit(site.USER_BASE.startswith("/tmp"))'],
             env=env)
-        self.assertEqual(rc, 1)
+        self.assertEqual(rc, 1,
+                        "User base not set by PYTHONUSERBASE")
 
     def test_getuserbase(self):
         site.USER_BASE = None

Makefile.pre.in

@@ -349,7 +349,7 @@ PYTHON_OBJS=	\
 		Python/pystate.o \
 		Python/pythonrun.o \
 		Python/pytime.o \
-		Python/random.o \
+		Python/bootstrap_hash.o \
 		Python/structmember.o \
 		Python/symtable.o \
 		Python/sysmodule.o \

Misc/NEWS

@@ -154,6 +154,8 @@ Core and Builtins
 - Issue #28596: The preferred encoding is UTF-8 on Android. Patch written by
   Chi Hsuan Yen.
 
+- bpo-22257: Clean up interpreter startup (see PEP 432).
+
 - Issue #26919: On Android, operating system data is now always encoded/decoded
   to/from UTF-8, instead of the locale encoding to avoid inconsistencies with
   os.fsencode() and os.fsdecode() which are already using UTF-8.

Modules/main.c

@@ -343,49 +343,44 @@ run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf)
 
 /* Main program */
 
-int
+/*TODO: Add arg processing to PEP 432 as a new configuration setup API
-Py_Main(int argc, wchar_t **argv)
+ */
+typedef struct {
+    wchar_t *filename;           /* Trailing arg without -c or -m */
+    wchar_t *command;            /* -c argument */
+    wchar_t *module;             /* -m argument */
+    PyObject *warning_options;   /* -W options */
+    PyObject *extra_options;     /* -X options */
+    int print_help;              /* -h, -? options */
+    int print_version;           /* -V option */
+    int bytes_warning;           /* Py_BytesWarningFlag */
+    int debug;                   /* Py_DebugFlag */
+    int inspect;                 /* Py_InspectFlag */
+    int interactive;             /* Py_InteractiveFlag */
+    int isolated;                /* Py_IsolatedFlag */
+    int optimization_level;      /* Py_OptimizeFlag */
+    int dont_write_bytecode;     /* Py_DontWriteBytecodeFlag */
+    int no_user_site_directory;  /* Py_NoUserSiteDirectory */
+    int no_site_import;          /* Py_NoSiteFlag */
+    int use_unbuffered_io;       /* Py_UnbufferedStdioFlag */
+    int verbosity;               /* Py_VerboseFlag */
+    int quiet_flag;              /* Py_QuietFlag */
+    int skip_first_line;         /* -x option */
+} _Py_CommandLineDetails;
+
+#define _Py_CommandLineDetails_INIT \
+            {NULL, NULL, NULL, NULL, NULL, \
+             0, 0, 0, 0, 0, 0, 0, 0, \
+             0, 0, 0, 0, 0, 0, 0}
+
+static int
+read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline)
 {
-    int c;
-    int sts;
-    wchar_t *command = NULL;
-    wchar_t *filename = NULL;
-    wchar_t *module = NULL;
-    FILE *fp = stdin;
-    char *p;
-#ifdef MS_WINDOWS
-    wchar_t *wp;
-#endif
-    int skipfirstline = 0;
-    int stdin_is_interactive = 0;
-    int help = 0;
-    int version = 0;
-    int saw_unbuffered_flag = 0;
-    char *opt;
-    PyCompilerFlags cf;
-    PyObject *main_importer_path = NULL;
     PyObject *warning_option = NULL;
-    PyObject *warning_options = NULL;
+    wchar_t *command = NULL;
-
+    wchar_t *module = NULL;
-    cf.cf_flags = 0;
+    char c;
-
+    char *opt;
-    orig_argc = argc;           /* For Py_GetArgcArgv() */
-    orig_argv = argv;
-
-    /* Hash randomization needed early for all string operations
-       (including -W and -X options). */
-    _PyOS_opterr = 0;  /* prevent printing the error in 1st pass */
-    while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) {
-        if (c == 'm' || c == 'c') {
-            /* -c / -m is the last option: following arguments are
-               not interpreter options. */
-            break;
-        }
-        if (c == 'E') {
-            Py_IgnoreEnvironmentFlag++;
-            break;
-        }
-    }
 
     opt = Py_GETENV("PYTHONMALLOC");
     if (_PyMem_SetupAllocators(opt) < 0) {
@@ -394,10 +389,11 @@ Py_Main(int argc, wchar_t **argv)
         exit(1);
     }
 
+	// TODO: Move these to core runtime init.
     Py_HashRandomizationFlag = 1;
-    _PyRandom_Init();
+    _Py_HashRandomization_Init();
-
     PySys_ResetWarnOptions();
+
     _PyOS_ResetGetOpt();
 
     while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) {
@@ -415,6 +411,7 @@ Py_Main(int argc, wchar_t **argv)
             wcscpy(command, _PyOS_optarg);
             command[len - 2] = '\n';
             command[len - 1] = 0;
+            cmdline->command = command;
             break;
         }
 
@@ -423,49 +420,49 @@ Py_Main(int argc, wchar_t **argv)
                that look like options are left for the
                module to interpret. */
             module = _PyOS_optarg;
+            cmdline->module = module;
             break;
         }
 
         switch (c) {
         case 'b':
-            Py_BytesWarningFlag++;
+            cmdline->bytes_warning++;
             break;
 
         case 'd':
-            Py_DebugFlag++;
+            cmdline->debug++;
             break;
 
         case 'i':
-            Py_InspectFlag++;
+            cmdline->inspect++;
-            Py_InteractiveFlag++;
+            cmdline->interactive++;
             break;
 
         case 'I':
-            Py_IsolatedFlag++;
+            cmdline->isolated++;
-            Py_NoUserSiteDirectory++;
+            cmdline->no_user_site_directory++;
-            Py_IgnoreEnvironmentFlag++;
             break;
 
         /* case 'J': reserved for Jython */
 
         case 'O':
-            Py_OptimizeFlag++;
+            cmdline->optimization_level++;
             break;
 
         case 'B':
-            Py_DontWriteBytecodeFlag++;
+            cmdline->dont_write_bytecode++;
             break;
 
         case 's':
-            Py_NoUserSiteDirectory++;
+            cmdline->no_user_site_directory++;
             break;
 
         case 'S':
-            Py_NoSiteFlag++;
+            cmdline->no_site_import++;
             break;
 
         case 'E':
-            /* Already handled above */
+            /* Handled prior to core initialization */
             break;
 
         case 't':
@@ -473,46 +470,46 @@ Py_Main(int argc, wchar_t **argv)
             break;
 
         case 'u':
-            Py_UnbufferedStdioFlag = 1;
+            cmdline->use_unbuffered_io = 1;
-            saw_unbuffered_flag = 1;
             break;
 
         case 'v':
-            Py_VerboseFlag++;
+            cmdline->verbosity++;
             break;
 
         case 'x':
-            skipfirstline = 1;
+            cmdline->skip_first_line = 1;
             break;
 
         case 'h':
         case '?':
-            help++;
+            cmdline->print_help++;
             break;
 
         case 'V':
-            version++;
+            cmdline->print_version++;
             break;
 
         case 'W':
-            if (warning_options == NULL)
+            if (cmdline->warning_options == NULL)
-                warning_options = PyList_New(0);
+                cmdline->warning_options = PyList_New(0);
-            if (warning_options == NULL)
+            if (cmdline->warning_options == NULL)
                 Py_FatalError("failure in handling of -W argument");
             warning_option = PyUnicode_FromWideChar(_PyOS_optarg, -1);
             if (warning_option == NULL)
                 Py_FatalError("failure in handling of -W argument");
-            if (PyList_Append(warning_options, warning_option) == -1)
+            if (PyList_Append(cmdline->warning_options, warning_option) == -1)
                 Py_FatalError("failure in handling of -W argument");
             Py_DECREF(warning_option);
             break;
 
         case 'X':
+            /* TODO: Delay addition of X options to sys module */
             PySys_AddXOption(_PyOS_optarg);
             break;
 
         case 'q':
-            Py_QuietFlag++;
+            cmdline->quiet_flag++;
             break;
 
         case 'R':
@@ -522,30 +519,110 @@ Py_Main(int argc, wchar_t **argv)
         /* This space reserved for other options */
 
         default:
-            return usage(2, argv[0]);
+            return -1;
             /*NOTREACHED*/
 
         }
     }
 
-    if (help)
+    if (command == NULL && module == NULL && _PyOS_optind < argc &&
-        return usage(0, argv[0]);
+        wcscmp(argv[_PyOS_optind], L"-") != 0)
+    {
+        cmdline->filename = argv[_PyOS_optind];
+    }
+    return 0;
+}
 
-    if (version) {
+static int
-        printf("Python %s\n", version >= 2 ? Py_GetVersion() : PY_VERSION);
+apply_command_line_and_environment(_Py_CommandLineDetails *cmdline)
+{
+    char *p;
+    Py_BytesWarningFlag = cmdline->bytes_warning;
+    Py_DebugFlag = cmdline->debug;
+    Py_InspectFlag = cmdline->inspect;
+    Py_InteractiveFlag = cmdline->interactive;
+    Py_IsolatedFlag = cmdline->isolated;
+    Py_OptimizeFlag = cmdline->optimization_level;
+    Py_DontWriteBytecodeFlag = cmdline->dont_write_bytecode;
+    Py_NoUserSiteDirectory = cmdline->no_user_site_directory;
+    Py_NoSiteFlag = cmdline->no_site_import;
+    Py_UnbufferedStdioFlag = cmdline->use_unbuffered_io;
+    Py_VerboseFlag = cmdline->verbosity;
+    Py_QuietFlag = cmdline->quiet_flag;
+
+    if (!Py_InspectFlag &&
+        (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') {
+        Py_InspectFlag = 1;
+        cmdline->inspect = 1;
+    }
+    if (!cmdline->use_unbuffered_io &&
+        (p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0') {
+        Py_UnbufferedStdioFlag = 1;
+        cmdline->use_unbuffered_io = 1;
+    }
+
+    if (!Py_NoUserSiteDirectory &&
+        (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0') {
+        Py_NoUserSiteDirectory = 1;
+        cmdline->no_user_site_directory = 1;
+    }
+
+    /* TODO: Apply PYTHONWARNINGS & -W options to sys module here */
+    /* TODO: Apply -X options to sys module here */
+    return 0;
+}
+
+int
+Py_Main(int argc, wchar_t **argv)
+{
+    int c;
+    int sts;
+    FILE *fp = stdin;
+    char *p;
+#ifdef MS_WINDOWS
+    wchar_t *wp;
+#endif
+    int stdin_is_interactive = 0;
+    _Py_CommandLineDetails cmdline = _Py_CommandLineDetails_INIT;
+    PyCompilerFlags cf;
+    PyObject *main_importer_path = NULL;
+
+    cf.cf_flags = 0;
+
+    orig_argc = argc;           /* For Py_GetArgcArgv() */
+    orig_argv = argv;
+
+    /* Hash randomization needed early for all string operations
+       (including -W and -X options). */
+    _PyOS_opterr = 0;  /* prevent printing the error in 1st pass */
+    while ((c = _PyOS_GetOpt(argc, argv, PROGRAM_OPTS)) != EOF) {
+        if (c == 'm' || c == 'c') {
+            /* -c / -m is the last option: following arguments are
+               not interpreter options. */
+            break;
+        }
+        if (c == 'E' || c == 'I') {
+            Py_IgnoreEnvironmentFlag++;
+            break;
+        }
+    }
+
+    /* Reprocess the command line with the language runtime available */
+    if (read_command_line(argc, argv, &cmdline)) {
+        return usage(2, argv[0]);
+    }
+
+    if (cmdline.print_help) {
+        return usage(0, argv[0]);
+    }
+
+    if (cmdline.print_version) {
+        printf("Python %s\n", cmdline.print_version >= 2 ? Py_GetVersion() : PY_VERSION);
         return 0;
     }
 
-    if (!Py_InspectFlag &&
+    PySys_ResetWarnOptions();
-        (p = Py_GETENV("PYTHONINSPECT")) && *p != '\0')
+    apply_command_line_and_environment(&cmdline);
-        Py_InspectFlag = 1;
-    if (!saw_unbuffered_flag &&
-        (p = Py_GETENV("PYTHONUNBUFFERED")) && *p != '\0')
-        Py_UnbufferedStdioFlag = 1;
-
-    if (!Py_NoUserSiteDirectory &&
-        (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0')
-        Py_NoUserSiteDirectory = 1;
 
 #ifdef MS_WINDOWS
     if (!Py_IgnoreEnvironmentFlag && (wp = _wgetenv(L"PYTHONWARNINGS")) &&
@@ -598,19 +675,13 @@ Py_Main(int argc, wchar_t **argv)
         PyMem_RawFree(buf);
     }
 #endif
-    if (warning_options != NULL) {
+    if (cmdline.warning_options != NULL) {
         Py_ssize_t i;
-        for (i = 0; i < PyList_GET_SIZE(warning_options); i++) {
+        for (i = 0; i < PyList_GET_SIZE(cmdline.warning_options); i++) {
-            PySys_AddWarnOptionUnicode(PyList_GET_ITEM(warning_options, i));
+            PySys_AddWarnOptionUnicode(PyList_GET_ITEM(cmdline.warning_options, i));
         }
     }
 
-    if (command == NULL && module == NULL && _PyOS_optind < argc &&
-        wcscmp(argv[_PyOS_optind], L"-") != 0)
-    {
-        filename = argv[_PyOS_optind];
-    }
-
     stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0);
 
 #if defined(MS_WINDOWS) || defined(__CYGWIN__)
@@ -697,31 +768,31 @@ Py_Main(int argc, wchar_t **argv)
     Py_SetProgramName(argv[0]);
 #endif
     Py_Initialize();
-    Py_XDECREF(warning_options);
+    Py_XDECREF(cmdline.warning_options);
 
     if (!Py_QuietFlag && (Py_VerboseFlag ||
-                        (command == NULL && filename == NULL &&
+                        (cmdline.command == NULL && cmdline.filename == NULL &&
-                         module == NULL && stdin_is_interactive))) {
+                         cmdline.module == NULL && stdin_is_interactive))) {
         fprintf(stderr, "Python %s on %s\n",
             Py_GetVersion(), Py_GetPlatform());
         if (!Py_NoSiteFlag)
             fprintf(stderr, "%s\n", COPYRIGHT);
     }
 
-    if (command != NULL) {
+    if (cmdline.command != NULL) {
         /* Backup _PyOS_optind and force sys.argv[0] = '-c' */
         _PyOS_optind--;
         argv[_PyOS_optind] = L"-c";
     }
 
-    if (module != NULL) {
+    if (cmdline.module != NULL) {
         /* Backup _PyOS_optind and force sys.argv[0] = '-m'*/
         _PyOS_optind--;
         argv[_PyOS_optind] = L"-m";
     }
 
-    if (filename != NULL) {
+    if (cmdline.filename != NULL) {
-        main_importer_path = AsImportPathEntry(filename);
+        main_importer_path = AsImportPathEntry(cmdline.filename);
     }
 
     if (main_importer_path != NULL) {
@@ -732,9 +803,11 @@ Py_Main(int argc, wchar_t **argv)
         PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
     }
 
-    if ((Py_InspectFlag || (command == NULL && filename == NULL && module == NULL)) &&
+    if ((Py_InspectFlag || (cmdline.command == NULL &&
-        isatty(fileno(stdin)) &&
+                            cmdline.filename == NULL &&
-        !Py_IsolatedFlag) {
+                            cmdline.module == NULL)) &&
+                            isatty(fileno(stdin)) &&
+                            !Py_IsolatedFlag) {
         PyObject *v;
         v = PyImport_ImportModule("readline");
         if (v == NULL)
@@ -743,15 +816,15 @@ Py_Main(int argc, wchar_t **argv)
             Py_DECREF(v);
     }
 
-    if (command) {
+    if (cmdline.command) {
-        sts = run_command(command, &cf);
+        sts = run_command(cmdline.command, &cf);
-        PyMem_RawFree(command);
+        PyMem_RawFree(cmdline.command);
-    } else if (module) {
+    } else if (cmdline.module) {
-        sts = (RunModule(module, 1) != 0);
+        sts = (RunModule(cmdline.module, 1) != 0);
     }
     else {
 
-        if (filename == NULL && stdin_is_interactive) {
+        if (cmdline.filename == NULL && stdin_is_interactive) {
             Py_InspectFlag = 0; /* do exit on SystemExit */
             RunStartupFile(&cf);
             RunInteractiveHook();
@@ -764,13 +837,13 @@ Py_Main(int argc, wchar_t **argv)
             sts = RunMainFromImporter(main_importer_path);
         }
 
-        if (sts==-1 && filename != NULL) {
+        if (sts==-1 && cmdline.filename != NULL) {
-            fp = _Py_wfopen(filename, L"r");
+            fp = _Py_wfopen(cmdline.filename, L"r");
             if (fp == NULL) {
                 char *cfilename_buffer;
                 const char *cfilename;
                 int err = errno;
-                cfilename_buffer = Py_EncodeLocale(filename, NULL);
+                cfilename_buffer = Py_EncodeLocale(cmdline.filename, NULL);
                 if (cfilename_buffer != NULL)
                     cfilename = cfilename_buffer;
                 else
@@ -781,7 +854,7 @@ Py_Main(int argc, wchar_t **argv)
                     PyMem_Free(cfilename_buffer);
                 return 2;
             }
-            else if (skipfirstline) {
+            else if (cmdline.skip_first_line) {
                 int ch;
                 /* Push back first newline so line numbers
                    remain the same */
@@ -798,7 +871,7 @@ Py_Main(int argc, wchar_t **argv)
                     S_ISDIR(sb.st_mode)) {
                     fprintf(stderr,
                             "%ls: '%ls' is a directory, cannot continue\n",
-                            argv[0], filename);
+                            argv[0], cmdline.filename);
                     fclose(fp);
                     return 1;
                 }
@@ -806,7 +879,7 @@ Py_Main(int argc, wchar_t **argv)
         }
 
         if (sts == -1)
-            sts = run_file(fp, filename, &cf);
+            sts = run_file(fp, cmdline.filename, &cf);
     }
 
     /* Check this environment variable at the end, to give programs the
@@ -819,7 +892,7 @@ Py_Main(int argc, wchar_t **argv)
     }
 
     if (Py_InspectFlag && stdin_is_interactive &&
-        (filename != NULL || command != NULL || module != NULL)) {
+        (cmdline.filename != NULL || cmdline.command != NULL || cmdline.module != NULL)) {
         Py_InspectFlag = 0;
         RunInteractiveHook();
         /* XXX */

PCbuild/pythoncore.vcxproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup Label="ProjectConfigurations">
     <ProjectConfiguration Include="Debug|Win32">
@@ -349,11 +349,11 @@
     <ClCompile Include="..\PC\getpathp.c" />
     <ClCompile Include="..\PC\msvcrtmodule.c" />
     <ClCompile Include="..\Python\pyhash.c" />
-    <ClCompile Include="..\Python\random.c" />
     <ClCompile Include="..\Python\_warnings.c" />
     <ClCompile Include="..\Python\asdl.c" />
     <ClCompile Include="..\Python\ast.c" />
     <ClCompile Include="..\Python\bltinmodule.c" />
+    <ClCompile Include="..\Python\bootstrap_hash.c" />
     <ClCompile Include="..\Python\ceval.c" />
     <ClCompile Include="..\Python\codecs.c" />
     <ClCompile Include="..\Python\compile.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
     <Filter Include="Include">
@@ -971,7 +971,7 @@
     <ClCompile Include="..\Python\traceback.c">
       <Filter>Python</Filter>
     </ClCompile>
-    <ClCompile Include="..\Python\random.c">
+    <ClCompile Include="..\Python\bootstrap_hash.c">
       <Filter>Python</Filter>
     </ClCompile>
     <ClCompile Include="..\Modules\_winapi.c">

Python/bootstrap_hash.c

@@ -533,44 +533,57 @@ _PyOS_URandomNonblock(void *buffer, Py_ssize_t size)
     return pyurandom(buffer, size, 0, 1);
 }
 
-void
+int Py_ReadHashSeed(char *seed_text,
-_PyRandom_Init(void)
+                    int *use_hash_seed,
+                    unsigned long *hash_seed)
 {
-    char *env;
-    unsigned char *secret = (unsigned char *)&_Py_HashSecret.uc;
-    Py_ssize_t secret_size = sizeof(_Py_HashSecret_t);
     Py_BUILD_ASSERT(sizeof(_Py_HashSecret_t) == sizeof(_Py_HashSecret.uc));
+    /* Convert a text seed to a numeric one */
+    if (seed_text && *seed_text != '\0' && strcmp(seed_text, "random") != 0) {
+        char *endptr = seed_text;
+        unsigned long seed;
+        seed = strtoul(seed_text, &endptr, 10);
+        if (*endptr != '\0'
+            || seed > 4294967295UL
+            || (errno == ERANGE && seed == ULONG_MAX))
+        {
+            return -1;
+        }
+        /* Use a specific hash */
+        *use_hash_seed = 1;
+        *hash_seed = seed;
+    }
+    else {
+        /* Use a random hash */
+        *use_hash_seed = 0;
+        *hash_seed = 0;
+    }
+    return 0;
+}
+
+static void
+init_hash_secret(int use_hash_seed,
+                 unsigned long hash_seed)
+{
+    void *secret = &_Py_HashSecret;
+    Py_ssize_t secret_size = sizeof(_Py_HashSecret_t);
 
     if (_Py_HashSecret_Initialized)
         return;
     _Py_HashSecret_Initialized = 1;
 
-    /*
+    if (use_hash_seed) {
-      Hash randomization is enabled.  Generate a per-process secret,
+        if (hash_seed == 0) {
-      using PYTHONHASHSEED if provided.
-    */
-
-    env = Py_GETENV("PYTHONHASHSEED");
-    if (env && *env != '\0' && strcmp(env, "random") != 0) {
-        char *endptr = env;
-        unsigned long seed;
-        seed = strtoul(env, &endptr, 10);
-        if (*endptr != '\0'
-            || seed > 4294967295UL
-            || (errno == ERANGE && seed == ULONG_MAX))
-        {
-            Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer "
-                          "in range [0; 4294967295]");
-        }
-        if (seed == 0) {
             /* disable the randomized hash */
             memset(secret, 0, secret_size);
         }
         else {
-            lcg_urandom(seed, secret, secret_size);
+            /* use the specified hash seed */
+            lcg_urandom(hash_seed, secret, secret_size);
         }
     }
     else {
+        /* use a random hash seed */
         int res;
 
         /* _PyRandom_Init() is called very early in the Python initialization
@@ -586,7 +599,24 @@ _PyRandom_Init(void)
 }
 
 void
-_PyRandom_Fini(void)
+_Py_HashRandomization_Init(void)
+{
+    char *seed_text;
+    int use_hash_seed = -1;
+    unsigned long hash_seed;
+
+    if (use_hash_seed < 0) {
+        seed_text = Py_GETENV("PYTHONHASHSEED");
+        if (Py_ReadHashSeed(seed_text, &use_hash_seed, &hash_seed) < 0) {
+            Py_FatalError("PYTHONHASHSEED must be \"random\" or an integer "
+                          "in range [0; 4294967295]");
+        }
+    }
+    init_hash_secret(use_hash_seed, hash_seed);
+}
+
+void
+_Py_HashRandomization_Fini(void)
 {
 #ifdef MS_WINDOWS
     if (hCryptProv) {

Python/pylifecycle.c

@@ -291,6 +291,9 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
 
     /* Install importlib as the implementation of import */
     value = PyObject_CallMethod(importlib, "_install", "OO", sysmod, impmod);
+    if (value != NULL)
+        value = PyObject_CallMethod(importlib,
+                                    "_install_external_importers", "");
     if (value == NULL) {
         PyErr_Print();
         Py_FatalError("Py_Initialize: importlib install failed");
@@ -331,8 +334,8 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
         Py_OptimizeFlag = add_flag(Py_OptimizeFlag, p);
     if ((p = Py_GETENV("PYTHONDONTWRITEBYTECODE")) && *p != '\0')
         Py_DontWriteBytecodeFlag = add_flag(Py_DontWriteBytecodeFlag, p);
-    /* The variable is only tested for existence here; _PyRandom_Init will
+    /* The variable is only tested for existence here;
-       check its value further. */
+       _Py_HashRandomization_Init will check its value further. */
     if ((p = Py_GETENV("PYTHONHASHSEED")) && *p != '\0')
         Py_HashRandomizationFlag = add_flag(Py_HashRandomizationFlag, p);
 #ifdef MS_WINDOWS
@@ -342,7 +345,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
         Py_LegacyWindowsStdioFlag = add_flag(Py_LegacyWindowsStdioFlag, p);
 #endif
 
-    _PyRandom_Init();
+    _Py_HashRandomization_Init();
 
     _PyInterpreterState_Init();
     interp = PyInterpreterState_New();
@@ -402,13 +405,15 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     /* initialize builtin exceptions */
     _PyExc_Init(bimod);
 
-    sysmod = _PySys_Init();
+    sysmod = _PySys_BeginInit();
     if (sysmod == NULL)
         Py_FatalError("Py_Initialize: can't initialize sys");
     interp->sysdict = PyModule_GetDict(sysmod);
     if (interp->sysdict == NULL)
         Py_FatalError("Py_Initialize: can't initialize sys dict");
     Py_INCREF(interp->sysdict);
+    if (_PySys_EndInit(interp->sysdict) < 0)
+        Py_FatalError("Py_Initialize: can't initialize sys");
     _PyImport_FixupBuiltin(sysmod, "sys");
     PySys_SetPath(Py_GetPath());
     PyDict_SetItemString(interp->sysdict, "modules",
@@ -694,7 +699,7 @@ Py_FinalizeEx(void)
     PyDict_Fini();
     PySlice_Fini();
     _PyGC_Fini();
-    _PyRandom_Fini();
+    _Py_HashRandomization_Fini();
     _PyArg_Fini();
     PyAsyncGen_Fini();
 

Python/pystate.c

@@ -46,6 +46,7 @@ static PyThread_type_lock head_mutex = NULL; /* Protects interp->tstate_head */
 /* The single PyInterpreterState used by this process'
    GILState implementation
 */
+/* TODO: Given interp_main, it may be possible to kill this ref */
 static PyInterpreterState *autoInterpreterState = NULL;
 static int autoTLSkey = -1;
 #else
@@ -55,6 +56,7 @@ static int autoTLSkey = -1;
 #endif
 
 static PyInterpreterState *interp_head = NULL;
+static PyInterpreterState *interp_main = NULL;
 
 /* Assuming the current thread holds the GIL, this is the
    PyThreadState for the current thread. */
@@ -119,6 +121,9 @@ PyInterpreterState_New(void)
 
         HEAD_LOCK();
         interp->next = interp_head;
+        if (interp_main == NULL) {
+            interp_main = interp;
+        }
         interp_head = interp;
         if (_next_interp_id < 0) {
             /* overflow or Py_Initialize() not called! */
@@ -185,6 +190,11 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
     if (interp->tstate_head != NULL)
         Py_FatalError("PyInterpreterState_Delete: remaining threads");
     *p = interp->next;
+    if (interp_main == interp) {
+        interp_main = NULL;
+        if (interp_head != NULL)
+            Py_FatalError("PyInterpreterState_Delete: remaining subinterpreters");
+    }
     HEAD_UNLOCK();
     PyMem_RawFree(interp);
 #ifdef WITH_THREAD
@@ -661,6 +671,12 @@ PyInterpreterState_Head(void)
     return interp_head;
 }
 
+PyInterpreterState *
+PyInterpreterState_Main(void)
+{
+    return interp_main;
+}
+
 PyInterpreterState *
 PyInterpreterState_Next(PyInterpreterState *interp) {
     return interp->next;

Python/sysmodule.c

@@ -1900,16 +1900,7 @@ static struct PyModuleDef sysmodule = {
     NULL
 };
 
-PyObject *
+/* Updating the sys namespace, returning NULL pointer on error */
-_PySys_Init(void)
-{
-    PyObject *m, *sysdict, *version_info;
-    int res;
-
-    m = PyModule_Create(&sysmodule);
-    if (m == NULL)
-        return NULL;
-    sysdict = PyModule_GetDict(m);
 #define SET_SYS_FROM_STRING_BORROW(key, value)             \
     do {                                                   \
         PyObject *v = (value);                             \
@@ -1932,6 +1923,17 @@ _PySys_Init(void)
         }                                                  \
     } while (0)
 
+PyObject *
+_PySys_BeginInit(void)
+{
+    PyObject *m, *sysdict, *version_info;
+    int res;
+
+    m = PyModule_Create(&sysmodule);
+    if (m == NULL)
+        return NULL;
+    sysdict = PyModule_GetDict(m);
+
     /* Check that stdin is not a directory
     Using shell redirection, you can redirect stdin to a directory,
     crashing the Python interpreter. Catch this common mistake here
@@ -1963,25 +1965,12 @@ _PySys_Init(void)
     SET_SYS_FROM_STRING("_git",
                         Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(),
                                       _Py_gitversion()));
-    SET_SYS_FROM_STRING("dont_write_bytecode",
-                         PyBool_FromLong(Py_DontWriteBytecodeFlag));
     SET_SYS_FROM_STRING("api_version",
                         PyLong_FromLong(PYTHON_API_VERSION));
     SET_SYS_FROM_STRING("copyright",
                         PyUnicode_FromString(Py_GetCopyright()));
     SET_SYS_FROM_STRING("platform",
                         PyUnicode_FromString(Py_GetPlatform()));
-    SET_SYS_FROM_STRING("executable",
-                        PyUnicode_FromWideChar(
-                               Py_GetProgramFullPath(), -1));
-    SET_SYS_FROM_STRING("prefix",
-                        PyUnicode_FromWideChar(Py_GetPrefix(), -1));
-    SET_SYS_FROM_STRING("exec_prefix",
-                        PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
-    SET_SYS_FROM_STRING("base_prefix",
-                        PyUnicode_FromWideChar(Py_GetPrefix(), -1));
-    SET_SYS_FROM_STRING("base_exec_prefix",
-                        PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
     SET_SYS_FROM_STRING("maxsize",
                         PyLong_FromSsize_t(PY_SSIZE_T_MAX));
     SET_SYS_FROM_STRING("float_info",
@@ -2017,17 +2006,6 @@ _PySys_Init(void)
     SET_SYS_FROM_STRING("abiflags",
                         PyUnicode_FromString(ABIFLAGS));
 #endif
-    if (warnoptions == NULL) {
-        warnoptions = PyList_New(0);
-        if (warnoptions == NULL)
-            return NULL;
-    }
-    else {
-        Py_INCREF(warnoptions);
-    }
-    SET_SYS_FROM_STRING_BORROW("warnoptions", warnoptions);
-
-    SET_SYS_FROM_STRING_BORROW("_xoptions", get_xoptions());
 
     /* version_info */
     if (VersionInfoType.tp_name == NULL) {
@@ -2052,13 +2030,8 @@ _PySys_Init(void)
         if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0)
             return NULL;
     }
+    /* Set flags to their default values */
     SET_SYS_FROM_STRING("flags", make_flags());
-    /* prevent user from creating new instances */
-    FlagsType.tp_init = NULL;
-    FlagsType.tp_new = NULL;
-    res = PyDict_DelItemString(FlagsType.tp_dict, "__new__");
-    if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError))
-        PyErr_Clear();
 
 #if defined(MS_WINDOWS)
     /* getwindowsversion */
@@ -2095,13 +2068,89 @@ _PySys_Init(void)
         }
     }
 
-#undef SET_SYS_FROM_STRING
-#undef SET_SYS_FROM_STRING_BORROW
     if (PyErr_Occurred())
         return NULL;
     return m;
 }
 
+#undef SET_SYS_FROM_STRING
+#undef SET_SYS_FROM_STRING_BORROW
+
+/* Updating the sys namespace, returning integer error codes */
+#define SET_SYS_FROM_STRING_BORROW_INT_RESULT(key, value)  \
+    do {                                                   \
+        PyObject *v = (value);                             \
+        if (v == NULL)                                     \
+            return -1;                                     \
+        res = PyDict_SetItemString(sysdict, key, v);       \
+        if (res < 0) {                                     \
+            return res;                                    \
+        }                                                  \
+    } while (0)
+#define SET_SYS_FROM_STRING_INT_RESULT(key, value)         \
+    do {                                                   \
+        PyObject *v = (value);                             \
+        if (v == NULL)                                     \
+            return -1;                                     \
+        res = PyDict_SetItemString(sysdict, key, v);       \
+        Py_DECREF(v);                                      \
+        if (res < 0) {                                     \
+            return res;                                    \
+        }                                                  \
+    } while (0)
+
+int
+_PySys_EndInit(PyObject *sysdict)
+{
+    int res;
+
+    /* Set flags to their final values */
+    SET_SYS_FROM_STRING_INT_RESULT("flags", make_flags());
+    /* prevent user from creating new instances */
+    FlagsType.tp_init = NULL;
+    FlagsType.tp_new = NULL;
+    res = PyDict_DelItemString(FlagsType.tp_dict, "__new__");
+    if (res < 0) {
+        if (!PyErr_ExceptionMatches(PyExc_KeyError)) {
+            return res;
+        }
+        PyErr_Clear();
+    }
+
+    SET_SYS_FROM_STRING_INT_RESULT("dont_write_bytecode",
+                         PyBool_FromLong(Py_DontWriteBytecodeFlag));
+    SET_SYS_FROM_STRING_INT_RESULT("executable",
+                        PyUnicode_FromWideChar(
+                               Py_GetProgramFullPath(), -1));
+    SET_SYS_FROM_STRING_INT_RESULT("prefix",
+                        PyUnicode_FromWideChar(Py_GetPrefix(), -1));
+    SET_SYS_FROM_STRING_INT_RESULT("exec_prefix",
+                        PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
+    SET_SYS_FROM_STRING_INT_RESULT("base_prefix",
+                        PyUnicode_FromWideChar(Py_GetPrefix(), -1));
+    SET_SYS_FROM_STRING_INT_RESULT("base_exec_prefix",
+                        PyUnicode_FromWideChar(Py_GetExecPrefix(), -1));
+
+    if (warnoptions == NULL) {
+        warnoptions = PyList_New(0);
+        if (warnoptions == NULL)
+            return -1;
+    }
+    else {
+        Py_INCREF(warnoptions);
+    }
+    SET_SYS_FROM_STRING_BORROW_INT_RESULT("warnoptions", warnoptions);
+
+    SET_SYS_FROM_STRING_BORROW_INT_RESULT("_xoptions", get_xoptions());
+
+    if (PyErr_Occurred())
+        return -1;
+    return 0;
+}
+
+#undef SET_SYS_FROM_STRING_INT_RESULT
+#undef SET_SYS_FROM_STRING_BORROW_INT_RESULT
+
 static PyObject *
 makepathobject(const wchar_t *path, wchar_t delim)
 {