diff --git a/scripts/fuzz-parser/fuzz.py b/scripts/fuzz-parser/fuzz.py index eb28056993..6e2da9ee54 100644 --- a/scripts/fuzz-parser/fuzz.py +++ b/scripts/fuzz-parser/fuzz.py @@ -11,8 +11,9 @@ Example invocations of the script: but only reporting bugs that are new on your branch: `python scripts/fuzz-parser/fuzz.py 0-10 --new-bugs-only` - Run the fuzzer concurrently on 10,000 different Python source-code files, - and only print a summary at the end: - `python scripts/fuzz-parser/fuzz.py 1-10000 --quiet + using a random selection of seeds, and only print a summary at the end + (the `shuf` command is Unix-specific): + `python scripts/fuzz-parser/fuzz.py $(shuf -i 0-1000000 -n 10000) --quiet """ from __future__ import annotations @@ -27,6 +28,7 @@ from typing import NewType from pysource_codegen import generate as generate_random_code from pysource_minimize import minimize as minimize_repro +from rich_argparse import RawDescriptionRichHelpFormatter from termcolor import colored MinimizedSourceCode = NewType("MinimizedSourceCode", str) @@ -67,19 +69,20 @@ class FuzzResult: # required to trigger the bug. If not, it will be `None`. maybe_bug: MinimizedSourceCode | None - def print_description(self) -> None: + def print_description(self, index: int, num_seeds: int) -> None: """Describe the results of fuzzing the parser with this seed.""" + progress = f"[{index}/{num_seeds}]" + msg = ( + colored(f"Ran fuzzer on seed {self.seed}", "red") + if self.maybe_bug + else colored(f"Ran fuzzer successfully on seed {self.seed}", "green") + ) + print(f"{msg:<55} {progress:>15}", flush=True) if self.maybe_bug: - print(colored(f"Ran fuzzer on seed {self.seed}", "red")) print(colored("The following code triggers a bug:", "red")) print() print(self.maybe_bug) print(flush=True) - else: - print( - colored(f"Ran fuzzer successfully on seed {self.seed}", "green"), - flush=True, - ) def fuzz_code( @@ -110,9 +113,10 @@ def fuzz_code( def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]: + num_seeds = len(args.seeds) print( f"Concurrently running the fuzzer on " - f"{len(args.seeds)} randomly generated source-code files..." + f"{num_seeds} randomly generated source-code files..." ) bugs: list[FuzzResult] = [] with concurrent.futures.ProcessPoolExecutor() as executor: @@ -127,10 +131,12 @@ def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]: for seed in args.seeds ] try: - for future in concurrent.futures.as_completed(fuzz_result_futures): + for i, future in enumerate( + concurrent.futures.as_completed(fuzz_result_futures), start=1 + ): fuzz_result = future.result() if not args.quiet: - fuzz_result.print_description() + fuzz_result.print_description(i, num_seeds) if fuzz_result.maybe_bug: bugs.append(fuzz_result) except KeyboardInterrupt: @@ -142,12 +148,13 @@ def run_fuzzer_concurrently(args: ResolvedCliArgs) -> list[FuzzResult]: def run_fuzzer_sequentially(args: ResolvedCliArgs) -> list[FuzzResult]: + num_seeds = len(args.seeds) print( f"Sequentially running the fuzzer on " - f"{len(args.seeds)} randomly generated source-code files..." + f"{num_seeds} randomly generated source-code files..." ) bugs: list[FuzzResult] = [] - for seed in args.seeds: + for i, seed in enumerate(args.seeds, start=1): fuzz_result = fuzz_code( seed, test_executable=args.test_executable, @@ -155,7 +162,7 @@ def run_fuzzer_sequentially(args: ResolvedCliArgs) -> list[FuzzResult]: only_new_bugs=args.only_new_bugs, ) if not args.quiet: - fuzz_result.print_description() + fuzz_result.print_description(i, num_seeds) if fuzz_result.maybe_bug: bugs.append(fuzz_result) return bugs @@ -212,7 +219,7 @@ class ResolvedCliArgs: def parse_args() -> ResolvedCliArgs: """Parse command-line arguments""" parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + description=__doc__, formatter_class=RawDescriptionRichHelpFormatter ) parser.add_argument( "seeds", @@ -293,7 +300,16 @@ def parse_args() -> ResolvedCliArgs: print( "Running `cargo build --release` since no test executable was specified..." ) - subprocess.run(["cargo", "build", "--release"], check=True, capture_output=True) + try: + subprocess.run( + ["cargo", "build", "--release", "--color", "always"], + check=True, + capture_output=True, + text=True, + ) + except subprocess.CalledProcessError as e: + print(e.stderr) + raise args.test_executable = os.path.join("target", "release", "ruff") assert os.path.exists(args.test_executable) diff --git a/scripts/fuzz-parser/requirements.in b/scripts/fuzz-parser/requirements.in index 6582e77e4d..b73747bdbc 100644 --- a/scripts/fuzz-parser/requirements.in +++ b/scripts/fuzz-parser/requirements.in @@ -1,4 +1,5 @@ pysource-codegen pysource-minimize +rich-argparse ruff termcolor diff --git a/scripts/fuzz-parser/requirements.txt b/scripts/fuzz-parser/requirements.txt index 781d7d6dcc..dcef183e00 100644 --- a/scripts/fuzz-parser/requirements.txt +++ b/scripts/fuzz-parser/requirements.txt @@ -12,11 +12,14 @@ mdurl==0.1.2 # via markdown-it-py pygments==2.17.2 # via rich -pysource-codegen==0.5.1 -pysource-minimize==0.6.2 +pysource-codegen==0.5.2 +pysource-minimize==0.6.3 rich==13.7.1 - # via pysource-minimize -ruff==0.4.0 + # via + # pysource-minimize + # rich-argparse +rich-argparse==1.4.0 +ruff==0.4.2 six==1.16.0 # via # asttokens