mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-18 16:55:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8
cpython/Tools/scripts/freeze_modules.py
Eric Snow 074fa57506
bpo-45395: Make custom frozen modules additions instead of replacements. (gh-28778) 
Currently custom modules (the array set on PyImport_FrozenModules) replace all the frozen stdlib modules. That can be problematic and is unlikely to be what the user wants. This change treats the custom frozen modules as additions instead. They take precedence over all other frozen modules except for those needed to bootstrap the import system. If the "code" field of an entry in the custom array is NULL then that frozen module is treated as disabled, which allows a custom entry to disable a frozen stdlib module.

This change allows us to get rid of is_essential_frozen_module() and simplifies the logic for which frozen modules should be ignored.

https://bugs.python.org/issue45395
2021-10-28 15:04:33 -06:00

763 lines
24 KiB
Python
Raw Blame History

"""Freeze modules and regen related files (e.g. Python/frozen.c).
See the notes at the top of Python/frozen.c for more info.
"""
from collections import namedtuple
import hashlib
import os
import ntpath
import posixpath
import platform
import subprocess
import sys
import textwrap
import time
from update_file import updating_file_with_tmpfile, update_file_with_tmpfile
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
ROOT_DIR = os.path.abspath(ROOT_DIR)
FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py')
STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
# If MODULES_DIR is changed then the .gitattributes and .gitignore files
# need to be updated.
MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
if sys.platform != "win32":
TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
if not os.path.isfile(TOOL):
# When building out of the source tree, get the tool from directory
# of the Python executable
TOOL = os.path.dirname(sys.executable)
TOOL = os.path.join(TOOL, 'Programs', '_freeze_module')
TOOL = os.path.abspath(TOOL)
if not os.path.isfile(TOOL):
sys.exit("ERROR: missing _freeze_module")
else:
def find_tool():
archs = ['amd64', 'win32']
if platform.machine() == "ARM64":
archs.append('arm64')
for arch in archs:
for exe in ['_freeze_module.exe', '_freeze_module_d.exe']:
tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe)
if os.path.isfile(tool):
return tool
sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat")
TOOL = find_tool()
del find_tool
MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST')
FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
PCBUILD_PROJECT = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj')
PCBUILD_FILTERS = os.path.join(ROOT_DIR, 'PCbuild', '_freeze_module.vcxproj.filters')
OS_PATH = 'ntpath' if os.name == 'nt' else 'posixpath'
# These are modules that get frozen.
TESTS_SECTION = 'Test module'
FROZEN = [
# See parse_frozen_spec() for the format.
# In cases where the frozenid is duplicated, the first one is re-used.
('import system', [
# These frozen modules are necessary for bootstrapping
# the import system.
'importlib._bootstrap : _frozen_importlib',
'importlib._bootstrap_external : _frozen_importlib_external',
# This module is important because some Python builds rely
# on a builtin zip file instead of a filesystem.
'zipimport',
]),
('stdlib - startup, without site (python -S)', [
'abc',
'codecs',
# For now we do not freeze the encodings, due # to the noise all
# those extra modules add to the text printed during the build.
# (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
#'<encodings.*>',
'io',
]),
('stdlib - startup, with site', [
'_collections_abc',
'_sitebuiltins',
'genericpath',
'ntpath',
'posixpath',
# We must explicitly mark os.path as a frozen module
# even though it will never be imported.
f'{OS_PATH} : os.path',
'os',
'site',
'stat',
]),
(TESTS_SECTION, [
'__hello__',
'__hello__ : __hello_alias__',
'__hello__ : <__phello_alias__>',
'__hello__ : __phello_alias__.spam',
'<__phello__.**.*>',
f'frozen_only : __hello_only__ = {FROZEN_ONLY}',
]),
]
BOOTSTRAP = {
'importlib._bootstrap',
'importlib._bootstrap_external',
'zipimport',
}
#######################################
# platform-specific helpers
if os.path is posixpath:
relpath_for_posix_display = os.path.relpath
def relpath_for_windows_display(path, base):
return ntpath.relpath(
ntpath.join(*path.split(os.path.sep)),
ntpath.join(*base.split(os.path.sep)),
)
else:
relpath_for_windows_display = ntpath.relpath
def relpath_for_posix_display(path, base):
return posixpath.relpath(
posixpath.join(*path.split(os.path.sep)),
posixpath.join(*base.split(os.path.sep)),
)
#######################################
# specs
def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
seen = {}
for section, specs in sectionalspecs:
parsed = _parse_specs(specs, section, seen)
for item in parsed:
frozenid, pyfile, modname, ispkg, section = item
try:
source = seen[frozenid]
except KeyError:
source = FrozenSource.from_id(frozenid, pyfile, destdir)
seen[frozenid] = source
else:
assert not pyfile or pyfile == source.pyfile, item
yield FrozenModule(modname, ispkg, section, source)
def _parse_specs(specs, section, seen):
for spec in specs:
info, subs = _parse_spec(spec, seen, section)
yield info
for info in subs or ():
yield info
def _parse_spec(spec, knownids=None, section=None):
"""Yield an info tuple for each module corresponding to the given spec.
The info consists of: (frozenid, pyfile, modname, ispkg, section).
Supported formats:
frozenid
frozenid : modname
frozenid : modname = pyfile
"frozenid" and "modname" must be valid module names (dot-separated
identifiers). If "modname" is not provided then "frozenid" is used.
If "pyfile" is not provided then the filename of the module
corresponding to "frozenid" is used.
Angle brackets around a frozenid (e.g. '<encodings>") indicate
it is a package. This also means it must be an actual module
(i.e. "pyfile" cannot have been provided). Such values can have
patterns to expand submodules:
<encodings.*> - also freeze all direct submodules
<encodings.**.*> - also freeze the full submodule tree
As with "frozenid", angle brackets around "modname" indicate
it is a package. However, in this case "pyfile" should not
have been provided and patterns in "modname" are not supported.
Also, if "modname" has brackets then "frozenid" should not,
and "pyfile" should have been provided..
"""
frozenid, _, remainder = spec.partition(':')
modname, _, pyfile = remainder.partition('=')
frozenid = frozenid.strip()
modname = modname.strip()
pyfile = pyfile.strip()
submodules = None
if modname.startswith('<') and modname.endswith('>'):
assert check_modname(frozenid), spec
modname = modname[1:-1]
assert check_modname(modname), spec
if frozenid in knownids:
pass
elif pyfile:
assert not os.path.isdir(pyfile), spec
else:
pyfile = _resolve_module(frozenid, ispkg=False)
ispkg = True
elif pyfile:
assert check_modname(frozenid), spec
assert not knownids or frozenid not in knownids, spec
assert check_modname(modname), spec
assert not os.path.isdir(pyfile), spec
ispkg = False
elif knownids and frozenid in knownids:
assert check_modname(frozenid), spec
assert check_modname(modname), spec
ispkg = False
else:
assert not modname or check_modname(modname), spec
resolved = iter(resolve_modules(frozenid))
frozenid, pyfile, ispkg = next(resolved)
if not modname:
modname = frozenid
if ispkg:
pkgid = frozenid
pkgname = modname
pkgfiles = {pyfile: pkgid}
def iter_subs():
for frozenid, pyfile, ispkg in resolved:
if pkgname:
modname = frozenid.replace(pkgid, pkgname, 1)
else:
modname = frozenid
if pyfile:
if pyfile in pkgfiles:
frozenid = pkgfiles[pyfile]
pyfile = None
elif ispkg:
pkgfiles[pyfile] = frozenid
yield frozenid, pyfile, modname, ispkg, section
submodules = iter_subs()
info = (frozenid, pyfile or None, modname, ispkg, section)
return info, submodules
#######################################
# frozen source files
class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):
@classmethod
def from_id(cls, frozenid, pyfile=None, destdir=MODULES_DIR):
if not pyfile:
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
#assert os.path.exists(pyfile), (frozenid, pyfile)
frozenfile = resolve_frozen_file(frozenid, destdir)
return cls(frozenid, pyfile, frozenfile)
@property
def frozenid(self):
return self.id
@property
def modname(self):
if self.pyfile.startswith(STDLIB_DIR):
return self.id
return None
@property
def symbol(self):
# This matches what we do in Programs/_freeze_module.c:
name = self.frozenid.replace('.', '_')
return '_Py_M__' + name
@property
def ispkg(self):
if not self.pyfile:
return False
elif self.frozenid.endswith('.__init__'):
return False
else:
return os.path.basename(self.pyfile) == '__init__.py'
def resolve_frozen_file(frozenid, destdir=MODULES_DIR):
"""Return the filename corresponding to the given frozen ID.
For stdlib modules the ID will always be the full name
of the source module.
"""
if not isinstance(frozenid, str):
try:
frozenid = frozenid.frozenid
except AttributeError:
raise ValueError(f'unsupported frozenid {frozenid!r}')
# We use a consistent naming convention for all frozen modules.
frozenfile = f'{frozenid}.h'
if not destdir:
return frozenfile
return os.path.join(destdir, frozenfile)
#######################################
# frozen modules
class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')):
def __getattr__(self, name):
return getattr(self.source, name)
@property
def modname(self):
return self.name
@property
def orig(self):
return self.source.modname
@property
def isalias(self):
orig = self.source.modname
if not orig:
return True
return self.name != orig
def summarize(self):
source = self.source.modname
if source:
source = f'<{source}>'
else:
source = relpath_for_posix_display(self.pyfile, ROOT_DIR)
return {
'module': self.name,
'ispkg': self.ispkg,
'source': source,
'frozen': os.path.basename(self.frozenfile),
'checksum': _get_checksum(self.frozenfile),
}
def _iter_sources(modules):
seen = set()
for mod in modules:
if mod.source not in seen:
yield mod.source
seen.add(mod.source)
#######################################
# generic helpers
def _get_checksum(filename):
with open(filename, "rb") as infile:
contents = infile.read()
m = hashlib.sha256()
m.update(contents)
return m.hexdigest()
def resolve_modules(modname, pyfile=None):
if modname.startswith('<') and modname.endswith('>'):
if pyfile:
assert os.path.isdir(pyfile) or os.path.basename(pyfile) == '__init__.py', pyfile
ispkg = True
modname = modname[1:-1]
rawname = modname
# For now, we only expect match patterns at the end of the name.
_modname, sep, match = modname.rpartition('.')
if sep:
if _modname.endswith('.**'):
modname = _modname[:-3]
match = f'**.{match}'
elif match and not match.isidentifier():
modname = _modname
# Otherwise it's a plain name so we leave it alone.
else:
match = None
else:
ispkg = False
rawname = modname
match = None
if not check_modname(modname):
raise ValueError(f'not a valid module name ({rawname})')
if not pyfile:
pyfile = _resolve_module(modname, ispkg=ispkg)
elif os.path.isdir(pyfile):
pyfile = _resolve_module(modname, pyfile, ispkg)
yield modname, pyfile, ispkg
if match:
pkgdir = os.path.dirname(pyfile)
yield from iter_submodules(modname, pkgdir, match)
def check_modname(modname):
return all(n.isidentifier() for n in modname.split('.'))
def iter_submodules(pkgname, pkgdir=None, match='*'):
if not pkgdir:
pkgdir = os.path.join(STDLIB_DIR, *pkgname.split('.'))
if not match:
match = '**.*'
match_modname = _resolve_modname_matcher(match, pkgdir)
def _iter_submodules(pkgname, pkgdir):
for entry in sorted(os.scandir(pkgdir), key=lambda e: e.name):
matched, recursive = match_modname(entry.name)
if not matched:
continue
modname = f'{pkgname}.{entry.name}'
if modname.endswith('.py'):
yield modname[:-3], entry.path, False
elif entry.is_dir():
pyfile = os.path.join(entry.path, '__init__.py')
# We ignore namespace packages.
if os.path.exists(pyfile):
yield modname, pyfile, True
if recursive:
yield from _iter_submodules(modname, entry.path)
return _iter_submodules(pkgname, pkgdir)
def _resolve_modname_matcher(match, rootdir=None):
if isinstance(match, str):
if match.startswith('**.'):
recursive = True
pat = match[3:]
assert match
else:
recursive = False
pat = match
if pat == '*':
def match_modname(modname):
return True, recursive
else:
raise NotImplementedError(match)
elif callable(match):
match_modname = match(rootdir)
else:
raise ValueError(f'unsupported matcher {match!r}')
return match_modname
def _resolve_module(modname, pathentry=STDLIB_DIR, ispkg=False):
assert pathentry, pathentry
pathentry = os.path.normpath(pathentry)
assert os.path.isabs(pathentry)
if ispkg:
return os.path.join(pathentry, *modname.split('.'), '__init__.py')
return os.path.join(pathentry, *modname.split('.')) + '.py'
#######################################
# regenerating dependent files
def find_marker(lines, marker, file):
for pos, line in enumerate(lines):
if marker in line:
return pos
raise Exception(f"Can't find {marker!r} in file {file}")
def replace_block(lines, start_marker, end_marker, replacements, file):
start_pos = find_marker(lines, start_marker, file)
end_pos = find_marker(lines, end_marker, file)
if end_pos <= start_pos:
raise Exception(f"End marker {end_marker!r} "
f"occurs before start marker {start_marker!r} "
f"in file {file}")
replacements = [line.rstrip() + '\n' for line in replacements]
return lines[:start_pos + 1] + replacements + lines[end_pos:]
def regen_manifest(modules):
header = 'module ispkg source frozen checksum'.split()
widths = [5] * len(header)
rows = []
for mod in modules:
info = mod.summarize()
row = []
for i, col in enumerate(header):
value = info[col]
if col == 'checksum':
value = value[:12]
elif col == 'ispkg':
value = 'YES' if value else 'no'
widths[i] = max(widths[i], len(value))
row.append(value or '-')
rows.append(row)
modlines = [
'# The list of frozen modules with key information.',
'# Note that the "check_generated_files" CI job will identify',
'# when source files were changed but regen-frozen wasn\'t run.',
'# This file is auto-generated by Tools/scripts/freeze_modules.py.',
' '.join(c.center(w) for c, w in zip(header, widths)).rstrip(),
' '.join('-' * w for w in widths),
]
for row in rows:
for i, w in enumerate(widths):
if header[i] == 'ispkg':
row[i] = row[i].center(w)
else:
row[i] = row[i].ljust(w)
modlines.append(' '.join(row).rstrip())
print(f'# Updating {os.path.relpath(MANIFEST)}')
with open(MANIFEST, 'w', encoding="utf-8") as outfile:
lines = (l + '\n' for l in modlines)
outfile.writelines(lines)
def regen_frozen(modules):
headerlines = []
parentdir = os.path.dirname(FROZEN_FILE)
for src in _iter_sources(modules):
# Adding a comment to separate sections here doesn't add much,
# so we don't.
header = relpath_for_posix_display(src.frozenfile, parentdir)
headerlines.append(f'#include "{header}"')
bootstraplines = []
stdliblines = []
testlines = []
aliaslines = []
indent = ' '
lastsection = None
for mod in modules:
if mod.frozenid in BOOTSTRAP:
lines = bootstraplines
elif mod.section == TESTS_SECTION:
lines = testlines
else:
lines = stdliblines
if mod.section != lastsection:
if lastsection is not None:
lines.append('')
lines.append(f'/* {mod.section} */')
lastsection = mod.section
symbol = mod.symbol
pkg = '-' if mod.ispkg else ''
line = ('{"%s", %s, %s(int)sizeof(%s)},'
) % (mod.name, symbol, pkg, symbol)
# TODO: Consider not folding lines
if len(line) < 80:
lines.append(line)
else:
line1, _, line2 = line.rpartition(' ')
lines.append(line1)
lines.append(indent + line2)
if mod.isalias:
if not mod.orig:
entry = '{"%s", NULL},' % (mod.name,)
elif mod.source.ispkg:
entry = '{"%s", "<%s"},' % (mod.name, mod.orig)
else:
entry = '{"%s", "%s"},' % (mod.name, mod.orig)
aliaslines.append(indent + entry)
for lines in (bootstraplines, stdliblines, testlines):
# TODO: Is this necessary any more?
if not lines[0]:
del lines[0]
for i, line in enumerate(lines):
if line:
lines[i] = indent + line
print(f'# Updating {os.path.relpath(FROZEN_FILE)}')
with updating_file_with_tmpfile(FROZEN_FILE) as (infile, outfile):
lines = infile.readlines()
# TODO: Use more obvious markers, e.g.
# $START GENERATED FOOBAR$ / $END GENERATED FOOBAR$
lines = replace_block(
lines,
"/* Includes for frozen modules: */",
"/* End includes */",
headerlines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"static const struct _frozen bootstrap_modules[] =",
"/* bootstrap sentinel */",
bootstraplines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"static const struct _frozen stdlib_modules[] =",
"/* stdlib sentinel */",
stdliblines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"static const struct _frozen test_modules[] =",
"/* test sentinel */",
testlines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"const struct _module_alias aliases[] =",
"/* aliases sentinel */",
aliaslines,
FROZEN_FILE,
)
outfile.writelines(lines)
def regen_makefile(modules):
pyfiles = []
frozenfiles = []
rules = ['']
for src in _iter_sources(modules):
header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
frozenfiles.append(f'\t\t{header} \\')
pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
pyfiles.append(f'\t\t{pyfile} \\')
freeze = (f'Programs/_freeze_module {src.frozenid} '
f'$(srcdir)/{pyfile} $(srcdir)/{header}')
rules.extend([
f'{header}: Programs/_freeze_module {pyfile}',
f'\t{freeze}',
'',
])
pyfiles[-1] = pyfiles[-1].rstrip(" \\")
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
print(f'# Updating {os.path.relpath(MAKEFILE)}')
with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
"FROZEN_FILES_IN =",
"# End FROZEN_FILES_IN",
pyfiles,
MAKEFILE,
)
lines = replace_block(
lines,
"FROZEN_FILES_OUT =",
"# End FROZEN_FILES_OUT",
frozenfiles,
MAKEFILE,
)
lines = replace_block(
lines,
"# BEGIN: freezing modules",
"# END: freezing modules",
rules,
MAKEFILE,
)
outfile.writelines(lines)
def regen_pcbuild(modules):
projlines = []
filterlines = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
projlines.append(f' <None Include="..\\{pyfile}">')
projlines.append(f' <ModName>{src.frozenid}</ModName>')
projlines.append(f' <IntFile>$(IntDir){intfile}</IntFile>')
projlines.append(f' <OutFile>$(PySourcePath){header}</OutFile>')
projlines.append(f' </None>')
filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>')
print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN frozen modules -->',
'<!-- END frozen modules -->',
projlines,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}')
with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN frozen modules -->',
'<!-- END frozen modules -->',
filterlines,
PCBUILD_FILTERS,
)
outfile.writelines(lines)
#######################################
# freezing modules
def freeze_module(modname, pyfile=None, destdir=MODULES_DIR):
"""Generate the frozen module .h file for the given module."""
tmpsuffix = f'.{int(time.time())}'
for modname, pyfile, ispkg in resolve_modules(modname, pyfile):
frozenfile = resolve_frozen_file(modname, destdir)
_freeze_module(modname, pyfile, frozenfile, tmpsuffix)
def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix):
tmpfile = f'{frozenfile}.{int(time.time())}'
print(tmpfile)
argv = [TOOL, frozenid, pyfile, tmpfile]
print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True)
try:
subprocess.run(argv, check=True)
except (FileNotFoundError, subprocess.CalledProcessError):
if not os.path.exists(TOOL):
sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"')
raise # re-raise
update_file_with_tmpfile(frozenfile, tmpfile, create=True)
#######################################
# the script
def main():
# Expand the raw specs, preserving order.
modules = list(parse_frozen_specs(destdir=MODULES_DIR))
# Regen build-related files.
regen_makefile(modules)
regen_pcbuild(modules)
# Freeze the target modules.
tmpsuffix = f'.{int(time.time())}'
for src in _iter_sources(modules):
_freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix)
# Regen files dependent of frozen file details.
regen_frozen(modules)
regen_manifest(modules)
if __name__ == '__main__':
argv = sys.argv[1:]
if argv:
sys.exit('ERROR: got unexpected args {argv}')
main()
Reference in a new issue View git blame Copy permalink