bpo-45020: Freeze some of the modules imported during startup. (gh-28335)

Doing this provides significant performance gains for runtime startup (~15% with all the imported modules frozen). We don't yet freeze all the imported modules because there are a few hiccups in the build systems we need to sort out first. (See bpo-45186 and bpo-45188.)

Note that in PR GH-28320 we added a command-line flag (-X frozen_modules=[on|off]) that allows users to opt out of (or into) using frozen modules. The default is still "off" but we will change it to "on" as soon as we can do it in a way that does not cause contributors pain.

https://bugs.python.org/issue45020

Lib/test/test_cmd_line_script.py

@@ -474,7 +474,6 @@ class CmdLineTest(unittest.TestCase):
                 br'ModuleNotFoundError'),
             ('builtins.x.y', br'Error while finding module specification.*'
                 br'ModuleNotFoundError.*No module named.*not a package'),
-            ('os.path', br'loader.*cannot handle'),
             ('importlib', br'No module named.*'
                 br'is a package and cannot be directly executed'),
             ('importlib.nonexistent', br'No module named'),

Lib/test/test_imp.py

@@ -16,6 +16,9 @@ with warnings.catch_warnings():
 import _imp
 
 
+OS_PATH_NAME = os.path.__name__
+
+
 def requires_load_dynamic(meth):
     """Decorator to skip a test if not running under CPython or lacking
     imp.load_dynamic()."""
@@ -213,15 +216,17 @@ class ImportTests(unittest.TestCase):
         # state after reversion. Reinitialising the module contents
         # and just reverting os.environ to its previous state is an OK
         # workaround
-        orig_path = os.path
-        orig_getenv = os.getenv
-        with os_helper.EnvironmentVarGuard():
-            x = imp.find_module("os")
-            self.addCleanup(x[0].close)
-            new_os = imp.load_module("os", *x)
-            self.assertIs(os, new_os)
-            self.assertIs(orig_path, new_os.path)
-            self.assertIsNot(orig_getenv, new_os.getenv)
+        with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
+            import os
+            orig_path = os.path
+            orig_getenv = os.getenv
+            with os_helper.EnvironmentVarGuard():
+                x = imp.find_module("os")
+                self.addCleanup(x[0].close)
+                new_os = imp.load_module("os", *x)
+                self.assertIs(os, new_os)
+                self.assertIs(orig_path, new_os.path)
+                self.assertIsNot(orig_getenv, new_os.getenv)
 
     @requires_load_dynamic
     def test_issue15828_load_extensions(self):

Lib/test/test_import/__init__.py

@@ -21,7 +21,7 @@ from unittest import mock
 from test.support import os_helper
 from test.support import (is_jython, swap_attr, swap_item, cpython_only)
 from test.support.import_helper import (
-    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath)
+    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
 from test.support.os_helper import (
     TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir)
 from test.support import script_helper
@@ -86,8 +86,10 @@ class ImportTests(unittest.TestCase):
             from importlib import something_that_should_not_exist_anywhere
 
     def test_from_import_missing_attr_has_name_and_path(self):
-        with self.assertRaises(ImportError) as cm:
-            from os import i_dont_exist
+        with CleanImport('os'):
+            import os
+            with self.assertRaises(ImportError) as cm:
+                from os import i_dont_exist
         self.assertEqual(cm.exception.name, 'os')
         self.assertEqual(cm.exception.path, os.__file__)
         self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)")

Lib/test/test_pydoc.py

@@ -23,6 +23,7 @@ import xml.etree.ElementTree
 import textwrap
 from io import StringIO
 from collections import namedtuple
+from test.support import import_helper
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test.support import threading_helper
@@ -728,6 +729,7 @@ class PydocDocTest(unittest.TestCase):
     @unittest.skipIf(sys.flags.optimize >= 2,
                      'Docstrings are omitted with -OO and above')
     def test_synopsis_sourceless(self):
+        os = import_helper.import_fresh_module('os')
         expected = os.__doc__.splitlines()[0]
         filename = os.__cached__
         synopsis = pydoc.synopsis(filename)

Lib/test/test_threading.py

@@ -1026,8 +1026,9 @@ class ThreadJoinOnShutdown(BaseTestCase):
 
             def random_io():
                 '''Loop for a while sleeping random tiny amounts and doing some I/O.'''
+                import test.test_threading as mod
                 while True:
-                    with open(os.__file__, 'rb') as in_f:
+                    with open(mod.__file__, 'rb') as in_f:
                         stuff = in_f.read(200)
                         with open(os.devnull, 'wb') as null_f:
                             null_f.write(stuff)

Lib/test/test_trace.py

@@ -376,7 +376,7 @@ class TestCoverage(unittest.TestCase):
 
     def test_coverage_ignore(self):
         # Ignore all files, nothing should be traced nor printed
-        libpath = os.path.normpath(os.path.dirname(os.__file__))
+        libpath = os.path.normpath(os.path.dirname(os.path.dirname(__file__)))
         # sys.prefix does not work when running from a checkout
         tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
                              libpath], trace=0, count=1)