bpo-45020: Freeze os, site, and codecs. (gh-28398)

https://bugs.python.org/issue45020

Lib/test/support/__init__.py

@@ -99,6 +99,13 @@ SHORT_TIMEOUT = 30.0
 # option.
 LONG_TIMEOUT = 5 * 60.0
 
+# TEST_HOME_DIR refers to the top level directory of the "test" package
+# that contains Python's regression test suite
+TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
+TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
+STDLIB_DIR = os.path.dirname(TEST_HOME_DIR)
+REPO_ROOT = os.path.dirname(STDLIB_DIR)
+
 
 class Error(Exception):
     """Base class for regression test exceptions."""
@@ -148,9 +155,7 @@ def load_package_tests(pkg_dir, loader, standard_tests, pattern):
     """
     if pattern is None:
         pattern = "test*"
-    top_dir = os.path.dirname(              # Lib
+    top_dir = STDLIB_DIR
-                  os.path.dirname(              # test
-                      os.path.dirname(__file__)))   # support
     package_tests = loader.discover(start_dir=pkg_dir,
                                     top_level_dir=top_dir,
                                     pattern=pattern)
@@ -459,11 +464,6 @@ PGO = False
 # PGO task.  If this is True, PGO is also True.
 PGO_EXTENDED = False
 
-# TEST_HOME_DIR refers to the top level directory of the "test" package
-# that contains Python's regression test suite
-TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
-TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
-
 # TEST_DATA_DIR is used as a target download location for remote resources
 TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data")
 
@@ -1390,7 +1390,7 @@ class PythonSymlink:
             self._env = {k.upper(): os.getenv(k) for k in os.environ}
             self._env["PYTHONHOME"] = os.path.dirname(self.real)
             if sysconfig.is_python_build(True):
-                self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
+                self._env["PYTHONPATH"] = STDLIB_DIR
 
     def __enter__(self):
         os.symlink(self.real, self.link)

Lib/test/test_embed.py

@@ -8,6 +8,7 @@ from collections import namedtuple
 import contextlib
 import json
 import os
+import os.path
 import re
 import shutil
 import subprocess
@@ -52,15 +53,13 @@ def remove_python_envvars():
 
 class EmbeddingTestsMixin:
     def setUp(self):
-        here = os.path.abspath(__file__)
-        basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
         exename = "_testembed"
         if MS_WINDOWS:
             ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
             exename += ext
             exepath = os.path.dirname(sys.executable)
         else:
-            exepath = os.path.join(basepath, "Programs")
+            exepath = os.path.join(support.REPO_ROOT, "Programs")
         self.test_exe = exe = os.path.join(exepath, exename)
         if not os.path.exists(exe):
             self.skipTest("%r doesn't exist" % exe)
@@ -68,7 +67,7 @@ class EmbeddingTestsMixin:
         # "Py_Initialize: Unable to get the locale encoding
         # LookupError: no codec search functions registered: can't find encoding"
         self.oldcwd = os.getcwd()
-        os.chdir(basepath)
+        os.chdir(support.REPO_ROOT)
 
     def tearDown(self):
         os.chdir(self.oldcwd)
@@ -1304,7 +1303,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 lib_dynload = os.path.join(pyvenv_home, 'lib')
                 os.makedirs(lib_dynload)
                 # getpathp.c uses Lib\os.py as the LANDMARK
-                shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
+                shutil.copyfile(
+                    os.path.join(support.STDLIB_DIR, 'os.py'),
+                    os.path.join(lib_dynload, 'os.py'),
+                )
 
             filename = os.path.join(tmpdir, 'pyvenv.cfg')
             with open(filename, "w", encoding="utf8") as fp:

Lib/test/test_import/__init__.py

@@ -19,7 +19,8 @@ import unittest
 from unittest import mock
 
 from test.support import os_helper
-from test.support import (is_jython, swap_attr, swap_item, cpython_only)
+from test.support import (
+    STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only)
 from test.support.import_helper import (
     forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
 from test.support.os_helper import (
@@ -495,7 +496,7 @@ class ImportTests(unittest.TestCase):
 
             env = None
             env = {k.upper(): os.environ[k] for k in os.environ}
-            env["PYTHONPATH"] = tmp2 + ";" + os.path.dirname(os.__file__)
+            env["PYTHONPATH"] = tmp2 + ";" + STDLIB_DIR
 
             # Test 1: import with added DLL directory
             subprocess.check_call([

Lib/test/test_site.py

@@ -591,7 +591,7 @@ class _pthFileTests(unittest.TestCase):
         return sys_path
 
     def test_underpth_nosite_file(self):
-        libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+        libpath = test.support.STDLIB_DIR
         exe_prefix = os.path.dirname(sys.executable)
         pth_lines = [
             'fake-path-name',
@@ -619,7 +619,7 @@ class _pthFileTests(unittest.TestCase):
         )
 
     def test_underpth_file(self):
-        libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+        libpath = test.support.STDLIB_DIR
         exe_prefix = os.path.dirname(sys.executable)
         exe_file = self._create_underpth_exe([
             'fake-path-name',
@@ -644,7 +644,7 @@ class _pthFileTests(unittest.TestCase):
 
 
     def test_underpth_dll_file(self):
-        libpath = os.path.dirname(os.path.dirname(encodings.__file__))
+        libpath = test.support.STDLIB_DIR
         exe_prefix = os.path.dirname(sys.executable)
         exe_file = self._create_underpth_exe([
             'fake-path-name',

Makefile.pre.in

@@ -762,6 +762,9 @@ Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
 Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
 	Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
 
+Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py
+	Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h
+
 Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
 	Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
 
@@ -780,6 +783,12 @@ Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
 Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
 	Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
 
+Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py
+	Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h
+
+Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py
+	Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h
+
 Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
 	Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
 
@@ -1023,12 +1032,15 @@ FROZEN_FILES = \
 		Python/frozen_modules/importlib__bootstrap_external.h \
 		Python/frozen_modules/zipimport.h \
 		Python/frozen_modules/abc.h \
+		Python/frozen_modules/codecs.h \
 		Python/frozen_modules/io.h \
 		Python/frozen_modules/_collections_abc.h \
 		Python/frozen_modules/_sitebuiltins.h \
 		Python/frozen_modules/genericpath.h \
 		Python/frozen_modules/ntpath.h \
 		Python/frozen_modules/posixpath.h \
+		Python/frozen_modules/os.h \
+		Python/frozen_modules/site.h \
 		Python/frozen_modules/stat.h \
 		Python/frozen_modules/__hello__.h
 # End FROZEN_FILES

PCbuild/_freeze_module.vcxproj

@@ -250,6 +250,11 @@
       <IntFile>$(IntDir)abc.g.h</IntFile>
       <OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
     </None>
+    <None Include="..\Lib\codecs.py">
+      <ModName>codecs</ModName>
+      <IntFile>$(IntDir)codecs.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
+    </None>
     <None Include="..\Lib\io.py">
       <ModName>io</ModName>
       <IntFile>$(IntDir)io.g.h</IntFile>
@@ -280,6 +285,16 @@
       <IntFile>$(IntDir)posixpath.g.h</IntFile>
       <OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
     </None>
+    <None Include="..\Lib\os.py">
+      <ModName>os</ModName>
+      <IntFile>$(IntDir)os.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
+    </None>
+    <None Include="..\Lib\site.py">
+      <ModName>site</ModName>
+      <IntFile>$(IntDir)site.g.h</IntFile>
+      <OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
+    </None>
     <None Include="..\Lib\stat.py">
       <ModName>stat</ModName>
       <IntFile>$(IntDir)stat.g.h</IntFile>

PCbuild/_freeze_module.vcxproj.filters

@@ -28,6 +28,9 @@
     <None Include="..\Lib\abc.py">
       <Filter>Python Files</Filter>
     </None>
+    <None Include="..\Lib\codecs.py">
+      <Filter>Python Files</Filter>
+    </None>
     <None Include="..\Lib\io.py">
       <Filter>Python Files</Filter>
     </None>
@@ -46,6 +49,12 @@
     <None Include="..\Lib\posixpath.py">
       <Filter>Python Files</Filter>
     </None>
+    <None Include="..\Lib\os.py">
+      <Filter>Python Files</Filter>
+    </None>
+    <None Include="..\Lib\site.py">
+      <Filter>Python Files</Filter>
+    </None>
     <None Include="..\Lib\stat.py">
       <Filter>Python Files</Filter>
     </None>

Python/frozen.c

@@ -42,12 +42,15 @@
 #include "frozen_modules/importlib__bootstrap_external.h"
 #include "frozen_modules/zipimport.h"
 #include "frozen_modules/abc.h"
+#include "frozen_modules/codecs.h"
 #include "frozen_modules/io.h"
 #include "frozen_modules/_collections_abc.h"
 #include "frozen_modules/_sitebuiltins.h"
 #include "frozen_modules/genericpath.h"
 #include "frozen_modules/ntpath.h"
 #include "frozen_modules/posixpath.h"
+#include "frozen_modules/os.h"
+#include "frozen_modules/site.h"
 #include "frozen_modules/stat.h"
 #include "frozen_modules/__hello__.h"
 /* End includes */
@@ -62,15 +65,21 @@ static const struct _frozen _PyImport_FrozenModules[] = {
         (int)sizeof(_Py_M__importlib__bootstrap_external)},
     {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
 
-    /* stdlib */
+    /* stdlib - startup, without site (python -S) */
     {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
+    {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs)},
     {"io", _Py_M__io, (int)sizeof(_Py_M__io)},
+
+    /* stdlib - startup, with site */
     {"_collections_abc", _Py_M___collections_abc,
         (int)sizeof(_Py_M___collections_abc)},
     {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
     {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
     {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
     {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
+    {"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
+    {"os", _Py_M__os, (int)sizeof(_Py_M__os)},
+    {"site", _Py_M__site, (int)sizeof(_Py_M__site)},
     {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
 
     /* Test module */

Tools/scripts/freeze_modules.py

@@ -66,16 +66,16 @@ FROZEN = [
         # on a builtin zip file instead of a filesystem.
         'zipimport',
         ]),
-    ('stdlib', [
+    ('stdlib - startup, without site (python -S)', [
-        # For the moment we skip codecs, encodings.*, os, and site.
-        # These modules have different generated files depending on
-        # if a debug or non-debug build.  (See bpo-45186 and bpo-45188.)
-        # without site (python -S)
         'abc',
-        #'codecs',
+        'codecs',
+        # For now we do not freeze the encodings, due # to the noise all
+        # those extra modules add to the text printed during the build.
+        # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
         #'<encodings.*>',
         'io',
-        # with site
+        ]),
+    ('stdlib - startup, with site', [
         '_collections_abc',
         '_sitebuiltins',
         'genericpath',
@@ -83,9 +83,9 @@ FROZEN = [
         'posixpath',
         # We must explicitly mark os.path as a frozen module
         # even though it will never be imported.
-        #f'{OS_PATH} : os.path',
+        f'{OS_PATH} : os.path',
-        #'os',
+        'os',
-        #'site',
+        'site',
         'stat',
         ]),
     ('Test module', [