mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-07-24 11:44:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

bpo-46712: Do not Regen Deep-Frozen Modules before Generating Global Objects (gh-32061)

Browse source 
We have to run "make regen-deepfreeze" before running Tools/scripts/generate-global-objects.py; otherwise we will miss any changes to global objects in deep-frozen modules (which aren't committed in the repo).  However, building $(PYTHON_FOR_FREEZE) fails if one of its source files had a global object (e.g. via _Py_ID(...)) added or removed, without generate-global-objects.py running first.  So "make regen-global-objects" would sometimes fail.

We solve this by running generate-global-objects.py before *and* after "make regen-deepfreeze". To speed things up and cut down on noise, we also avoid updating the global objects files if there are no changes to them.

https://bugs.python.org/issue46712
This commit is contained in:
Eric Snow 2022-03-23 09:55:52 -06:00 • committed by GitHub
parent 21412d037b
commit febf54bcf3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 36 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

7
Makefile.pre.in
View file

@ -1179,7 +1179,12 @@ regen-importlib: regen-frozen
# Global objects # Global objects
.PHONY: regen-global-objects .PHONY: regen-global-objects
regen-global-objects: regen-deepfreeze $(srcdir)/Tools/scripts/generate_global_objects.py regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
@# Run one more time after deepfreezing, to catch any globals added
@# there. This is necessary because the deep-frozen code isn't
@# commited to the repo.
$(MAKE) regen-deepfreeze
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
############################################################################ ############################################################################

41
Tools/scripts/generate_global_objects.py
View file

@ -1,5 +1,6 @@
import contextlib import contextlib
import glob import glob
import io
import os.path import os.path
import re import re
import sys import sys
@ -123,6 +124,7 @@ def iter_global_strings():
varname, string = m.groups() varname, string = m.groups()
yield varname, string, filename, lno, line yield varname, string, filename, lno, line
def iter_to_marker(lines, marker): def iter_to_marker(lines, marker):
for line in lines: for line in lines:
if line.rstrip() == marker: if line.rstrip() == marker:
@ -165,6 +167,19 @@ class Printer:
self.write("}" + suffix) self.write("}" + suffix)
@contextlib.contextmanager
def open_for_changes(filename, orig):
"""Like open() but only write to the file if it changed."""
outfile = io.StringIO()
yield outfile
text = outfile.getvalue()
if text != orig:
with open(filename, 'w', encoding='utf-8') as outfile:
outfile.write(text)
else:
print(f'# not changed: {filename}')
####################################### #######################################
# the global objects # the global objects
@ -177,13 +192,15 @@ def generate_global_strings(identifiers, strings):
# Read the non-generated part of the file. # Read the non-generated part of the file.
with open(filename) as infile: with open(filename) as infile:
before = ''.join(iter_to_marker(infile, START))[:-1] orig = infile.read()
for _ in iter_to_marker(infile, END): lines = iter(orig.rstrip().splitlines())
pass before = '\n'.join(iter_to_marker(lines, START))
after = infile.read()[:-1] for _ in iter_to_marker(lines, END):
pass
after = '\n'.join(lines)
# Generate the file. # Generate the file.
with open(filename, 'w', encoding='utf-8') as outfile: with open_for_changes(filename, orig) as outfile:
printer = Printer(outfile) printer = Printer(outfile)
printer.write(before) printer.write(before)
printer.write(START) printer.write(START)
@ -202,7 +219,6 @@ def generate_global_strings(identifiers, strings):
with printer.block('struct', ' latin1[128];'): with printer.block('struct', ' latin1[128];'):
printer.write("PyCompactUnicodeObject _latin1;") printer.write("PyCompactUnicodeObject _latin1;")
printer.write("uint8_t _data[2];") printer.write("uint8_t _data[2];")
printer.write(END) printer.write(END)
printer.write(after) printer.write(after)
@ -227,13 +243,15 @@ def generate_runtime_init(identifiers, strings):
# Read the non-generated part of the file. # Read the non-generated part of the file.
with open(filename) as infile: with open(filename) as infile:
before = ''.join(iter_to_marker(infile, START))[:-1] orig = infile.read()
for _ in iter_to_marker(infile, END): lines = iter(orig.rstrip().splitlines())
pass before = '\n'.join(iter_to_marker(lines, START))
after = infile.read()[:-1] for _ in iter_to_marker(lines, END):
pass
after = '\n'.join(lines)
# Generate the file. # Generate the file.
with open(filename, 'w', encoding='utf-8') as outfile: with open_for_changes(filename, orig) as outfile:
printer = Printer(outfile) printer = Printer(outfile)
printer.write(before) printer.write(before)
printer.write(START) printer.write(START)
@ -286,6 +304,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]':
raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}') raise ValueError(f'string mismatch for {name!r} ({string!r} != {strings[name]!r}')
return identifiers, strings return identifiers, strings
####################################### #######################################
# the script # the script