mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 18:29:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 10

gh-109515: When generating deep frozen modules on Windows, use a list file instead of arguments (GH-109516)

Browse source
This commit is contained in:
Riccardo Ghetta 2023-10-30 17:14:26 +01:00 • committed by GitHub
parent 6dfb8fe023
commit 8eaa206fec
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 78 additions and 30 deletions
Download patch file Download diff file Expand all files Collapse all files

64
PCbuild/_freeze_module.vcxproj
View file

@ -395,6 +395,33 @@
<OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile> <OutFile>$(PySourcePath)Python\frozen_modules\getpath.h</OutFile>
</GetPath> </GetPath>
</ItemGroup> </ItemGroup>
<ItemGroup>
<!-- BEGIN freeze mappings -->
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap.h" FrozenId="importlib._bootstrap" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib._bootstrap_external.h" FrozenId="importlib._bootstrap_external" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\zipimport.h" FrozenId="zipimport" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\abc.h" FrozenId="abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\codecs.h" FrozenId="codecs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\io.h" FrozenId="io" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_collections_abc.h" FrozenId="_collections_abc" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\_sitebuiltins.h" FrozenId="_sitebuiltins" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\genericpath.h" FrozenId="genericpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\ntpath.h" FrozenId="ntpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\posixpath.h" FrozenId="posixpath" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\os.h" FrozenId="os" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\site.h" FrozenId="site" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\stat.h" FrozenId="stat" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.util.h" FrozenId="importlib.util" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\importlib.machinery.h" FrozenId="importlib.machinery" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\runpy.h" FrozenId="runpy" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__hello__.h" FrozenId="__hello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.h" FrozenId="__phello__" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.h" FrozenId="__phello__.ham" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.ham.eggs.h" FrozenId="__phello__.ham.eggs" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\__phello__.spam.h" FrozenId="__phello__.spam" />
<FrozenModule Include="$(PySourcePath)\Python\frozen_modules\frozen_only.h" FrozenId="frozen_only" />
<!-- END freeze mappings -->
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets"> <ImportGroup Label="ExtensionTargets">
</ImportGroup> </ImportGroup>
@ -426,32 +453,19 @@
AfterTargets="_RebuildFrozen" AfterTargets="_RebuildFrozen"
DependsOnTargets="FindPythonForBuild" DependsOnTargets="FindPythonForBuild"
Condition="$(Configuration) != 'PGUpdate'"> Condition="$(Configuration) != 'PGUpdate'">
<PropertyGroup>
<FreezeMappingsFile>
<!-- BEGIN freeze mapping file -->
$(IntDir)\deepfreeze_mappings.txt
<!-- END freeze mapping file -->
</FreezeMappingsFile>
</PropertyGroup>
<WriteLinesToFile
File="$(FreezeMappingsFile)"
Overwrite="true"
Lines="@(FrozenModule->'%(FullPath):%(FrozenId)')" />
<!-- BEGIN deepfreeze rule --> <!-- BEGIN deepfreeze rule -->
<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" ^ <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" -f "$(IntDir)\deepfreeze_mappings.txt" -o "$(PySourcePath)Python\deepfreeze\deepfreeze.c"' />
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h:importlib._bootstrap" ^
"$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h:importlib._bootstrap_external" ^
"$(PySourcePath)Python\frozen_modules\zipimport.h:zipimport" ^
"$(PySourcePath)Python\frozen_modules\abc.h:abc" ^
"$(PySourcePath)Python\frozen_modules\codecs.h:codecs" ^
"$(PySourcePath)Python\frozen_modules\io.h:io" ^
"$(PySourcePath)Python\frozen_modules\_collections_abc.h:_collections_abc" ^
"$(PySourcePath)Python\frozen_modules\_sitebuiltins.h:_sitebuiltins" ^
"$(PySourcePath)Python\frozen_modules\genericpath.h:genericpath" ^
"$(PySourcePath)Python\frozen_modules\ntpath.h:ntpath" ^
"$(PySourcePath)Python\frozen_modules\posixpath.h:posixpath" ^
"$(PySourcePath)Python\frozen_modules\os.h:os" ^
"$(PySourcePath)Python\frozen_modules\site.h:site" ^
"$(PySourcePath)Python\frozen_modules\stat.h:stat" ^
"$(PySourcePath)Python\frozen_modules\importlib.util.h:importlib.util" ^
"$(PySourcePath)Python\frozen_modules\importlib.machinery.h:importlib.machinery" ^
"$(PySourcePath)Python\frozen_modules\runpy.h:runpy" ^
"$(PySourcePath)Python\frozen_modules\__hello__.h:__hello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.h:__phello__" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.h:__phello__.ham" ^
"$(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h:__phello__.ham.eggs" ^
"$(PySourcePath)Python\frozen_modules\__phello__.spam.h:__phello__.spam" ^
"$(PySourcePath)Python\frozen_modules\frozen_only.h:frozen_only" ^
"-o" "$(PySourcePath)Python\deepfreeze\deepfreeze.c"'/>
<!-- END deepfreeze rule --> <!-- END deepfreeze rule -->
</Target> </Target>
<Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'"> <Target Name="_CleanFrozen" BeforeTargets="CoreClean" Condition="$(Configuration) != 'PGUpdate'">

16
Tools/build/deepfreeze.py
View file

@ -488,7 +488,10 @@ def generate(args: list[str], output: TextIO) -> None:
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c") parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c")
parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics")
parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format") group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-f", "--file", help="read rule lines from a file")
group.add_argument('args', nargs="*", default=(),
help="Input file and module name (required) in file:modname format")
@contextlib.contextmanager @contextlib.contextmanager
def report_time(label: str): def report_time(label: str):
@ -506,9 +509,18 @@ def main() -> None:
args = parser.parse_args() args = parser.parse_args()
verbose = args.verbose verbose = args.verbose
output = args.output output = args.output
if args.file:
if verbose:
print(f"Reading targets from {args.file}")
with open(args.file, "rt", encoding="utf-8-sig") as fin:
rules = [x.strip() for x in fin]
else:
rules = args.args
with open(output, "w", encoding="utf-8") as file: with open(output, "w", encoding="utf-8") as file:
with report_time("generate"): with report_time("generate"):
generate(args.args, file) generate(rules, file)
if verbose: if verbose:
print(f"Wrote {os.path.getsize(output)} bytes to {output}") print(f"Wrote {os.path.getsize(output)} bytes to {output}")

28
Tools/build/freeze_modules.py
View file

@ -21,6 +21,7 @@ STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
# .gitattributes and .gitignore files needs to be updated. # .gitattributes and .gitignore files needs to be updated.
FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules') FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze') DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt'
FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c') FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in') MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
@ -645,7 +646,9 @@ def regen_pcbuild(modules):
projlines = [] projlines = []
filterlines = [] filterlines = []
corelines = [] corelines = []
deepfreezerules = ['\t<Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" ^'] deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
deepfreezerules = [f' <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\' />']
deepfreezemappings = []
for src in _iter_sources(modules): for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
@ -659,8 +662,7 @@ def regen_pcbuild(modules):
filterlines.append(f' <None Include="..\\{pyfile}">') filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>') filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>') filterlines.append(' </None>')
deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^') deepfreezemappings.append(f' <FrozenModule Include="$(PySourcePath)\\{header}" FrozenId="{src.frozenid}" />\n')
deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' )
corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />') corelines.append(f' <ClCompile Include="..\\Python\\deepfreeze\\deepfreeze.c" />')
@ -675,6 +677,26 @@ def regen_pcbuild(modules):
PCBUILD_PROJECT, PCBUILD_PROJECT,
) )
outfile.writelines(lines) outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mappings -->',
'<!-- END freeze mappings -->',
deepfreezemappings,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mapping file -->',
'<!-- END freeze mapping file -->',
[deepfreezemappingsfile, ],
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines() lines = infile.readlines()
lines = replace_block( lines = replace_block(