diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 68d234722e..ed1ac23bf6 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -33,12 +33,8 @@ jobs: - name: on current branch; prepare a self-contained benchmark folder run: nix develop -c ./ci/benchmarks/prep_folder.sh branch - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - name: build benchmark runner - run: nix develop -c `cd ci/benchmarks/bench-runner && cargo build --release && cd ../../..` + run: nix develop -c bash -c "cd ci/benchmarks/bench-runner && cargo build --release && cd ../../.." - name: run benchmarks with regression check run: nix develop -c ./ci/benchmarks/bench-runner/target/release/bench-runner --check-executables-changed diff --git a/Cargo.toml b/Cargo.toml index 6a5e0b066c..4106ed25cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,7 +58,7 @@ members = [ "crates/wasi-libc-sys", ] exclude = [ - "ci/bench-runner", + "ci/benchmarks/bench-runner", # Examples sometimes have Rust hosts in their platforms. The compiler should ignore those. "crates/cli_testing_examples", "examples", diff --git a/ci/benchmarks/bench-runner/src/main.rs b/ci/benchmarks/bench-runner/src/main.rs index 38afcd801d..7fbbd91979 100644 --- a/ci/benchmarks/bench-runner/src/main.rs +++ b/ci/benchmarks/bench-runner/src/main.rs @@ -33,7 +33,7 @@ fn main() { if check_if_bench_executables_changed() { println!( - "Comparison of sha256 of executables reveals changes, doing full benchmarks..." + "\n\nComparison of sha256 of executables reveals changes, doing full benchmarks...\n\n" ); let all_regressed_benches = do_all_benches(optional_args.nr_repeat_benchmarks); @@ -85,6 +85,8 @@ fn do_all_benches(nr_repeat_benchmarks: usize) -> HashSet { return HashSet::new(); } + println!("\n\nDoing benchmarks {:?} times to reduce flukes.\n\n", nr_repeat_benchmarks); + for _ in 1..nr_repeat_benchmarks { delete_old_bench_results(); do_benchmark("main"); @@ -112,7 +114,7 @@ fn do_benchmark(branch_name: &'static str) -> HashSet { )) .args(&["--bench", "--noplot"]) .stdout(Stdio::piped()) - .stderr(Stdio::piped()) + .stderr(Stdio::inherit()) .spawn() .unwrap_or_else(|_| panic!("Failed to benchmark {}.", branch_name)); @@ -133,14 +135,14 @@ fn do_benchmark(branch_name: &'static str) -> HashSet { "Failed to get line that contains benchmark name from last_three_lines_queue.", ); - let regex_match = bench_name_regex.find(regressed_bench_name_line).expect("This line should hoave the benchmark name between double quotes but I could not match it"); + let regex_match = bench_name_regex.find(regressed_bench_name_line).expect("This line should have the benchmark name between double quotes but I could not match it"); regressed_benches.insert(regex_match.as_str().to_string().replace("\"", "")); } last_three_lines_queue.push_front(line_str.clone()); - println!("bench {:?}: {:?}", branch_name, line_str); + println!(">>bench {:?}: {:?}", branch_name, line_str); } regressed_benches @@ -186,8 +188,20 @@ fn sha256_digest(mut reader: R) -> Result { } fn sha_file(file_path: &Path) -> Result { - let input = File::open(file_path)?; - let reader = BufReader::new(input); + // Debug info is dependent on the dir in which executable was created, + // so we need to strip that to be able to compare binaries. + let no_debug_info_file_path = file_path.to_str().unwrap().to_string() + ("_no_debug_info"); + std::fs::copy(file_path, &no_debug_info_file_path)?; + + let strip_output = Command::new("strip") + .args(["--strip-debug", &no_debug_info_file_path]) + .output() + .expect("failed to execute process"); + + assert!(strip_output.status.success()); + + let no_debug_info_file = File::open(no_debug_info_file_path)?; + let reader = BufReader::new(no_debug_info_file); let digest = sha256_digest(reader)?; Ok(HEXUPPER.encode(digest.as_ref())) @@ -241,6 +255,9 @@ fn check_if_bench_executables_changed() -> bool { if let Some(main_hash_val) = main_bench_hashes.get(key) { if let Some(branch_hash_val) = branch_bench_hashes.get(key) { if !main_hash_val.eq(branch_hash_val) { + dbg!(main_hash_val); + dbg!(branch_hash_val); + dbg!(key); return true; } } else { diff --git a/crates/cli_utils/src/bench_utils.rs b/crates/cli_utils/src/bench_utils.rs index 31da15a8e7..a49161b78b 100644 --- a/crates/cli_utils/src/bench_utils.rs +++ b/crates/cli_utils/src/bench_utils.rs @@ -2,7 +2,7 @@ use crate::helpers::{file_path_from_root, run_cmd, run_roc}; use criterion::{black_box, measurement::Measurement, BenchmarkGroup}; use std::{path::Path, thread}; -const CFOLD_STACK_SIZE: usize = 8192 * 100000; +const CFOLD_STACK_SIZE: usize = 16384 * 100000; const OPTIMIZE_FLAG: &str = "--optimize"; diff --git a/crates/compiler/build/src/link.rs b/crates/compiler/build/src/link.rs index 601a21bd56..48c40a29a6 100644 --- a/crates/compiler/build/src/link.rs +++ b/crates/compiler/build/src/link.rs @@ -143,7 +143,7 @@ pub fn build_zig_host_native( command.args(&[ zig_host_src, - emit_bin, + &format!("-femit-bin={}", emit_bin), "--pkg-begin", "str", zig_str_path, @@ -211,7 +211,7 @@ pub fn build_zig_host_native( command.args(&[ zig_host_src, - emit_bin, + &format!("-femit-bin={}", emit_bin), "--pkg-begin", "str", zig_str_path, @@ -308,7 +308,7 @@ pub fn build_zig_host_native( } command.args(&[ zig_host_src, - emit_bin, + &format!("-femit-bin={}", emit_bin), "--pkg-begin", "str", zig_str_path, @@ -583,8 +583,6 @@ pub fn rebuild_host( ) } Architecture::X86_64 => { - let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap()); - let target = match target.operating_system { OperatingSystem::Windows => "x86_64-windows-gnu", _ => "native", @@ -593,7 +591,7 @@ pub fn rebuild_host( build_zig_host_native( &env_path, &env_home, - &emit_bin, + host_dest.to_str().unwrap(), zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), target, @@ -601,33 +599,27 @@ pub fn rebuild_host( shared_lib_path, ) } - Architecture::X86_32(_) => { - let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap()); - build_zig_host_native( - &env_path, - &env_home, - &emit_bin, - zig_host_src.to_str().unwrap(), - zig_str_path.to_str().unwrap(), - "i386-linux-musl", - opt_level, - shared_lib_path, - ) - } + Architecture::X86_32(_) => build_zig_host_native( + &env_path, + &env_home, + host_dest.to_str().unwrap(), + zig_host_src.to_str().unwrap(), + zig_str_path.to_str().unwrap(), + "i386-linux-musl", + opt_level, + shared_lib_path, + ), - Architecture::Aarch64(_) => { - let emit_bin = format!("-femit-bin={}", host_dest.to_str().unwrap()); - build_zig_host_native( - &env_path, - &env_home, - &emit_bin, - zig_host_src.to_str().unwrap(), - zig_str_path.to_str().unwrap(), - target_zig_str(target), - opt_level, - shared_lib_path, - ) - } + Architecture::Aarch64(_) => build_zig_host_native( + &env_path, + &env_home, + host_dest.to_str().unwrap(), + zig_host_src.to_str().unwrap(), + zig_str_path.to_str().unwrap(), + target_zig_str(target), + opt_level, + shared_lib_path, + ), _ => internal_error!("Unsupported architecture {:?}", target.architecture), };