mirrors/cpython
1
0
Fork 0
mirror of https://github.com/python/cpython.git synced 2025-09-26 10:19:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

gh-104504: Run mypy on cases_generator in CI (and blacken the code) (gh-108090)

Browse source 
Co-authored-by: Alex Waygood <Alex.Waygood@Gmail.com>
This commit is contained in:
Dong-hee Na 2023-08-18 22:42:45 +09:00 • committed by GitHub
parent fd19509220
commit 28cab71f95
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 313 additions and 194 deletions
Download patch file Download diff file Expand all files Collapse all files

97
Tools/cases_generator/generate_cases.py
View file

@ -10,6 +10,7 @@ import os
import posixpath
import sys
import typing
from collections.abc import Iterator
import stacking # Early import to avoid circular import
from analysis import Analyzer
@ -23,7 +24,6 @@ from instructions import (
MacroInstruction,
MacroParts,
PseudoInstruction,
StackEffect,
OverriddenInstructionPlaceHolder,
TIER_ONE,
TIER_TWO,
@ -80,12 +80,10 @@ SPECIALLY_HANDLED_ABSTRACT_INSTR = {
"STORE_FAST",
"STORE_FAST_MAYBE_NULL",
"COPY",
# Arithmetic
"_BINARY_OP_MULTIPLY_INT",
"_BINARY_OP_ADD_INT",
"_BINARY_OP_SUBTRACT_INT",
}
arg_parser = argparse.ArgumentParser(
@ -144,6 +142,7 @@ arg_parser.add_argument(
default=DEFAULT_ABSTRACT_INTERPRETER_OUTPUT,
)
class Generator(Analyzer):
def get_stack_effect_info(
self, thing: parsing.InstDef | parsing.Macro | parsing.Pseudo
@ -183,7 +182,8 @@ class Generator(Analyzer):
assert target_instr
target_popped = effect_str(target_instr.input_effects)
target_pushed = effect_str(target_instr.output_effects)
if popped is None and pushed is None:
if pushed is None:
assert popped is None
popped, pushed = target_popped, target_pushed
else:
assert popped == target_popped
@ -193,7 +193,7 @@ class Generator(Analyzer):
return instr, popped, pushed
@contextlib.contextmanager
def metadata_item(self, signature, open, close):
def metadata_item(self, signature: str, open: str, close: str) -> Iterator[None]:
self.out.emit("")
self.out.emit(f"extern {signature};")
self.out.emit("#ifdef NEED_OPCODE_METADATA")
@ -216,9 +216,10 @@ class Generator(Analyzer):
def write_function(
direction: str, data: list[tuple[AnyInstruction, str]]
) -> None:
with self.metadata_item(
f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)", "", ""
f"int _PyOpcode_num_{direction}(int opcode, int oparg, bool jump)",
"",
"",
):
with self.out.block("switch(opcode)"):
for instr, effect in data:
@ -243,23 +244,24 @@ class Generator(Analyzer):
paths = f"\n{self.out.comment} ".join(filenames)
return f"{self.out.comment} from:\n{self.out.comment} {paths}\n"
def write_provenance_header(self):
def write_provenance_header(self) -> None:
self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n")
self.out.write_raw(self.from_source_files())
self.out.write_raw(f"{self.out.comment} Do not edit!\n")
def assign_opcode_ids(self):
def assign_opcode_ids(self) -> None:
"""Assign IDs to opcodes"""
ops: list[(bool, str)] = [] # (has_arg, name) for each opcode
ops: list[tuple[bool, str]] = [] # (has_arg, name) for each opcode
instrumented_ops: list[str] = []
for instr in itertools.chain(
[instr for instr in self.instrs.values() if instr.kind != "op"],
self.macro_instrs.values()):
self.macro_instrs.values(),
):
assert isinstance(instr, (Instruction, MacroInstruction, PseudoInstruction))
name = instr.name
if name.startswith('INSTRUMENTED_'):
if name.startswith("INSTRUMENTED_"):
instrumented_ops.append(name)
else:
ops.append((instr.instr_flags.HAS_ARG_FLAG, name))
@ -268,33 +270,32 @@ class Generator(Analyzer):
# rather than bytecodes.c, so we need to add it explicitly
# here (at least until we add something to bytecodes.c to
# declare external instructions).
instrumented_ops.append('INSTRUMENTED_LINE')
instrumented_ops.append("INSTRUMENTED_LINE")
# assert lists are unique
assert len(set(ops)) == len(ops)
assert len(set(instrumented_ops)) == len(instrumented_ops)
opname: list[str or None] = [None] * 512
opmap: dict = {}
markers: dict = {}
opname: list[str | None] = [None] * 512
opmap: dict[str, int] = {}
markers: dict[str, int] = {}
def map_op(op, name):
def map_op(op: int, name: str) -> None:
assert op < len(opname)
assert opname[op] is None
assert name not in opmap
opname[op] = name
opmap[name] = op
# 0 is reserved for cache entries. This helps debugging.
map_op(0, 'CACHE')
map_op(0, "CACHE")
# 17 is reserved as it is the initial value for the specializing counter.
# This helps catch cases where we attempt to execute a cache.
map_op(17, 'RESERVED')
map_op(17, "RESERVED")
# 166 is RESUME - it is hard coded as such in Tools/build/deepfreeze.py
map_op(166, 'RESUME')
map_op(166, "RESUME")
next_opcode = 1
@ -306,13 +307,13 @@ class Generator(Analyzer):
assert next_opcode < 255
map_op(next_opcode, name)
if has_arg and 'HAVE_ARGUMENT' not in markers:
markers['HAVE_ARGUMENT'] = next_opcode
if has_arg and "HAVE_ARGUMENT" not in markers:
markers["HAVE_ARGUMENT"] = next_opcode
# Instrumented opcodes are at the end of the valid range
min_instrumented = 254 - (len(instrumented_ops) - 1)
assert next_opcode <= min_instrumented
markers['MIN_INSTRUMENTED_OPCODE'] = min_instrumented
markers["MIN_INSTRUMENTED_OPCODE"] = min_instrumented
for i, op in enumerate(instrumented_ops):
map_op(min_instrumented + i, op)
@ -320,11 +321,13 @@ class Generator(Analyzer):
for i, op in enumerate(sorted(self.pseudos)):
map_op(256 + i, op)
assert 255 not in opmap # 255 is reserved
assert 255 not in opmap.values() # 255 is reserved
self.opmap = opmap
self.markers = markers
def write_opcode_ids(self, opcode_ids_h_filename, opcode_targets_filename):
def write_opcode_ids(
self, opcode_ids_h_filename: str, opcode_targets_filename: str
) -> None:
"""Write header file that defined the opcode IDs"""
with open(opcode_ids_h_filename, "w") as f:
@ -337,15 +340,15 @@ class Generator(Analyzer):
self.out.emit("#ifndef Py_OPCODE_IDS_H")
self.out.emit("#define Py_OPCODE_IDS_H")
self.out.emit("#ifdef __cplusplus")
self.out.emit("extern \"C\" {")
self.out.emit('extern "C" {')
self.out.emit("#endif")
self.out.emit("")
self.out.emit("/* Instruction opcodes for compiled code */")
def define(name, opcode):
def define(name: str, opcode: int) -> None:
self.out.emit(f"#define {name:<38} {opcode:>3}")
all_pairs = []
all_pairs: list[tuple[int, int, str]] = []
# the second item in the tuple sorts the markers before the ops
all_pairs.extend((i, 1, name) for (name, i) in self.markers.items())
all_pairs.extend((i, 2, name) for (name, i) in self.opmap.items())
@ -370,7 +373,6 @@ class Generator(Analyzer):
targets[op] = f"TARGET_{name}"
f.write(",\n".join([f" &&{s}" for s in targets]))
def write_metadata(self, metadata_filename: str, pymetadata_filename: str) -> None:
"""Write instruction metadata to output file."""
@ -469,7 +471,7 @@ class Generator(Analyzer):
"const struct opcode_metadata "
"_PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE]",
"=",
";"
";",
):
# Write metadata for each instruction
for thing in self.everything:
@ -482,7 +484,9 @@ class Generator(Analyzer):
case parsing.Macro():
self.write_metadata_for_macro(self.macro_instrs[thing.name])
case parsing.Pseudo():
self.write_metadata_for_pseudo(self.pseudo_instrs[thing.name])
self.write_metadata_for_pseudo(
self.pseudo_instrs[thing.name]
)
case _:
typing.assert_never(thing)
@ -490,7 +494,7 @@ class Generator(Analyzer):
"const struct opcode_macro_expansion "
"_PyOpcode_macro_expansion[OPCODE_MACRO_EXPANSION_SIZE]",
"=",
";"
";",
):
# Write macro expansion for each non-pseudo instruction
for thing in self.everything:
@ -529,7 +533,9 @@ class Generator(Analyzer):
self.write_uop_items(lambda name, counter: f'[{name}] = "{name}",')
with self.metadata_item(
f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]", "=", ";"
f"const char *const _PyOpcode_OpName[{1 + max(self.opmap.values())}]",
"=",
";",
):
for name in self.opmap:
self.out.emit(f'[{name}] = "{name}",')
@ -542,11 +548,9 @@ class Generator(Analyzer):
for m in family.members:
deoptcodes[m] = name
# special case:
deoptcodes['BINARY_OP_INPLACE_ADD_UNICODE'] = 'BINARY_OP'
deoptcodes["BINARY_OP_INPLACE_ADD_UNICODE"] = "BINARY_OP"
with self.metadata_item(
f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"
):
with self.metadata_item(f"const uint8_t _PyOpcode_Deopt[256]", "=", ";"):
for opt, deopt in sorted(deoptcodes.items()):
self.out.emit(f"[{opt}] = {deopt},")
@ -604,10 +608,9 @@ class Generator(Analyzer):
if name not in specialized_ops:
self.out.emit(f"'{name}': {op},")
for name in ['MIN_INSTRUMENTED_OPCODE', 'HAVE_ARGUMENT']:
for name in ["MIN_INSTRUMENTED_OPCODE", "HAVE_ARGUMENT"]:
self.out.emit(f"{name} = {self.markers[name]}")
def write_pseudo_instrs(self) -> None:
"""Write the IS_PSEUDO_INSTR macro"""
self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) ( \\")
@ -834,7 +837,10 @@ class Generator(Analyzer):
pass
case parsing.InstDef():
instr = AbstractInstruction(self.instrs[thing.name].inst)
if instr.is_viable_uop() and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR:
if (
instr.is_viable_uop()
and instr.name not in SPECIALLY_HANDLED_ABSTRACT_INSTR
):
self.out.emit("")
with self.out.block(f"case {thing.name}:"):
instr.write(self.out, tier=TIER_TWO)
@ -878,7 +884,7 @@ class Generator(Analyzer):
self.out.emit(f"DISPATCH();")
def main():
def main() -> None:
"""Parse command line, parse input, analyze, write output."""
args = arg_parser.parse_args() # Prints message and sys.exit(2) on error
if len(args.input) == 0:
@ -899,8 +905,9 @@ def main():
a.write_opcode_ids(args.opcode_ids_h, args.opcode_targets_h)
a.write_metadata(args.metadata, args.pymetadata)
a.write_executor_instructions(args.executor_cases, args.emit_line_directives)
a.write_abstract_interpreter_instructions(args.abstract_interpreter_cases,
args.emit_line_directives)
a.write_abstract_interpreter_instructions(
args.abstract_interpreter_cases, args.emit_line_directives
)
if __name__ == "__main__":