bpo-45019: Cleanup module freezing and deepfreeze (#29772)

Tools/scripts/freeze_modules.py

@@ -8,12 +8,9 @@ import hashlib
 import os
 import ntpath
 import posixpath
-import platform
-import subprocess
 import sys
-import time
 
-from update_file import updating_file_with_tmpfile, update_file_with_tmpfile
+from update_file import updating_file_with_tmpfile
 
 
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
@@ -21,9 +18,10 @@ ROOT_DIR = os.path.abspath(ROOT_DIR)
 FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
 
 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
-# If MODULES_DIR is changed then the .gitattributes and .gitignore files
+# If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the 
-# need to be updated.
+# .gitattributes and .gitignore files needs to be updated.
-MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
+FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
+DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
 
 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
 MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
@@ -111,16 +109,16 @@ else:
 #######################################
 # specs
 
-def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
+def parse_frozen_specs():
     seen = {}
-    for section, specs in sectionalspecs:
+    for section, specs in FROZEN:
         parsed = _parse_specs(specs, section, seen)
         for item in parsed:
             frozenid, pyfile, modname, ispkg, section = item
             try:
                 source = seen[frozenid]
             except KeyError:
-                source = FrozenSource.from_id(frozenid, pyfile, destdir)
+                source = FrozenSource.from_id(frozenid, pyfile)
                 seen[frozenid] = source
             else:
                 assert not pyfile or pyfile == source.pyfile, item
@@ -225,15 +223,16 @@ def _parse_spec(spec, knownids=None, section=None):
 #######################################
 # frozen source files
 
-class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
+class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):
 
     @classmethod
-    def from_id(cls, frozenid, pyfile=None, destdir=MODULES_DIR):
+    def from_id(cls, frozenid, pyfile=None):
         if not pyfile:
             pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
             #assert os.path.exists(pyfile), (frozenid, pyfile)
-        frozenfile = resolve_frozen_file(frozenid, destdir)
+        frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
-        return cls(frozenid, pyfile, frozenfile)
+        deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR)
+        return cls(frozenid, pyfile, frozenfile, deepfreezefile)
 
     @property
     def frozenid(self):
@@ -261,7 +260,7 @@ class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
             return os.path.basename(self.pyfile) == '__init__.py'
 
 
-def resolve_frozen_file(frozenid, destdir=MODULES_DIR):
+def resolve_frozen_file(frozenid, destdir):
     """Return the filename corresponding to the given frozen ID.
 
     For stdlib modules the ID will always be the full name
@@ -570,41 +569,30 @@ def regen_makefile(modules):
     deepfreezefiles = []
     rules = ['']
     deepfreezerules = ['']
-
-    # TODO: Merge the two loops
     for src in _iter_sources(modules):
-        header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
+        frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
-        relfile = header.replace('\\', '/')
+        deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR)
-        _pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
+        frozenfiles.append(f'\t\t{frozen_header} \\')
-
+        cfile = deepfreeze_header[:-2] + ".c"
-        # TODO: This is a bit hackish
+        ofile = deepfreeze_header[:-2] + ".o"
-        xfile = relfile.replace("/frozen_modules/", "/deepfreeze/")
-        cfile = xfile[:-2] + ".c"
-        ofile = xfile[:-2] + ".o"
         deepfreezefiles.append(f"\t\t{ofile} \\")
 
-        # Also add a deepfreeze rule.
-        deepfreezerules.append(f'{cfile}: {header} $(DEEPFREEZE_DEPS)')
-        deepfreezerules.append(
-            f"\t$(PYTHON_FOR_REGEN) "
-            f"$(srcdir)/Tools/scripts/deepfreeze.py "
-            f"{header} -m {src.frozenid} -o {cfile}")
-        deepfreezerules.append('')
-
-    for src in _iter_sources(modules):
-        header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
-        frozenfiles.append(f'\t\t{header} \\')
-
         pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
         pyfiles.append(f'\t\t{pyfile} \\')
 
         freeze = (f'$(FREEZE_MODULE) {src.frozenid} '
-                  f'$(srcdir)/{pyfile} {header}')
+                  f'$(srcdir)/{pyfile} {frozen_header}')
         rules.extend([
-            f'{header}: $(FREEZE_MODULE) {pyfile}',
+            f'{frozen_header}: $(FREEZE_MODULE) {pyfile}',
             f'\t{freeze}',
             '',
         ])
+        deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
+        deepfreezerules.append(
+            f"\t$(PYTHON_FOR_REGEN) "
+            f"$(srcdir)/Tools/scripts/deepfreeze.py "
+            f"{frozen_header} -m {src.frozenid} -o {cfile}")
+        deepfreezerules.append('')
     pyfiles[-1] = pyfiles[-1].rstrip(" \\")
     frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
     deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\")
@@ -715,7 +703,7 @@ def regen_pcbuild(modules):
 
 def main():
     # Expand the raw specs, preserving order.
-    modules = list(parse_frozen_specs(destdir=MODULES_DIR))
+    modules = list(parse_frozen_specs())
 
     # Regen build-related files.
     regen_makefile(modules)