started rewriting benchmark bash script in rust

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "roc_std",
     "docs",
 ]
+exclude = [ "ci/bench-runner" ]
 # Needed to be able to run `cargo run -p roc_cli --no-default-features` -
 # see www/build.sh for more.
 #

ci/bench-runner.sh

@@ -45,6 +45,12 @@ for ctr in `seq 1 $NR_REPEATS`;
 
         EXIT_CODE=$?
 
+        if  [ $EXIT_CODE -ne 0 ]; then
+            echo ""
+            echo "Benchmark execution failed with exit code: $EXIT_CODE."
+            echo ""
+        fi
+
         if [ $ctr -eq 1 ]; then
 
             rm -f bench_sha_sums.txt
@@ -54,14 +60,24 @@ for ctr in `seq 1 $NR_REPEATS`;
             find examples/benchmarks -executable -type f | while read exec_filename; do
                 sha1sum $exec_filename >> bench_sha_sums.txt
             done
+            echo "cat1"
+            cat bench_sha_sums.txt
 
             # on trunk
             find ../bench-folder-trunk/examples/benchmarks -executable -type f | while read exec_filename; do
                 sha1sum $exec_filename >> bench_sha_sums.txt
             done
+            echo "cat2"
+            cat bench_sha_sums.txt
 
             # keep only unique sha sums
+            echo "sort"
+            sort bench_sha_sums.txt
+            echo "sort with uniq"
+            sort bench_sha_sums.txt | uniq -u
             sort bench_sha_sums.txt | uniq -u > bench_sha_sums.txt
+            echo "cat3"
+            cat bench_sha_sums.txt
 
             if [ $(cat bench_sha_sums.txt | wc -l) -eq 0 ]; then
                 echo ""

ci/bench-runner/Cargo.lock

@@ -0,0 +1,255 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
+
+[[package]]
+name = "bench-runner"
+version = "0.1.0"
+dependencies = [
+ "clap",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
+
+[[package]]
+name = "clap"
+version = "3.0.0-beta.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142"
+dependencies = [
+ "atty",
+ "bitflags",
+ "clap_derive",
+ "indexmap",
+ "lazy_static",
+ "os_str_bytes",
+ "strsim",
+ "termcolor",
+ "textwrap",
+ "unicode-width",
+ "vec_map",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.0.0-beta.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
+
+[[package]]
+name = "os_str_bytes"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85"
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "1.0.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "unicode-segmentation"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
+
+[[package]]
+name = "unicode-width"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
+
+[[package]]
+name = "vec_map"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
+
+[[package]]
+name = "version_check"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"

ci/bench-runner/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "bench-runner"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clap = "3.0.0-beta.2"

ci/bench-runner/src/main.rs

@@ -0,0 +1,78 @@
+use std::{path::Path, process::{self, Command, Stdio}, thread};
+use clap::{AppSettings, Clap};
+
+
+fn main() {
+    let optional_args: OptionalArgs = OptionalArgs::parse();
+
+    if Path::new("bench-folder-trunk").exists() && Path::new("bench-folder-branch").exists() {
+
+        delete_old_bench_results();
+        
+        do_benchmark("trunk");
+        //do_benchmark("branch");
+
+        if optional_args.test_run {
+            //println!("Doing a test run to verify benchmarks are working correctly")
+        }
+    } else {
+        eprintln!(r#"I can't find bench-folder-trunk and bench-folder-branch from the current directory.
+        I should be executed from the repo root.
+        Use `./ci/safe-earthly.sh --build-arg BENCH_SUFFIX=trunk +prep-bench-folder` to generate bench-folder-trunk.
+        Use `./ci/safe-earthly.sh +prep-bench-folder` to generate bench-folder-branch."#);
+        
+        process::exit(1)
+    }
+}
+
+fn do_benchmark(branch_name: &'static str) {
+    /*let builder = thread::Builder::new()
+                  .name("reductor".into())
+                  .stack_size(32 * 1024 * 1024); // 32MB of stack space, necessary for cfold benchmark
+
+    let handler = builder.spawn(move || {
+        Command::new(format!("./bench-folder-{}/target/release/deps/time_bench", branch_name))
+        .args(&BENCH_ARGS)
+        .stdout(Stdio::inherit())
+        .output()
+        .expect(&format!("Failed to benchmark {}.", branch_name));
+    }).unwrap();
+    
+    handler.join().unwrap();*/
+
+    Command::new(format!("./bench-folder-{}/target/release/deps/time_bench", branch_name))
+        .args(&["--bench", "--noplot"])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .output()
+        .expect(&format!("Failed to benchmark {}.", branch_name));
+
+}
+
+fn delete_old_bench_results() {
+    remove("target/criterion");
+}
+
+// does not error if fileOrFolder does not exist (-f flag)
+fn remove(file_or_folder: &str) {
+    Command::new("rm")
+        .args(&["-rf", file_or_folder])
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit())
+        .output()
+        .expect(&format!("Something went wrong trying to remove {}", file_or_folder));
+}
+
+#[derive(Clap)]
+#[clap(setting = AppSettings::ColoredHelp)]
+struct OptionalArgs {
+    /// Do a test run: short warmup and few repeats to verify benchmarks are working correctly
+    #[clap(long)]
+    test_run: bool,
+    /// How many times to repeat the benchmarks. A single benchmark has to fail every for a regression to be reported.
+    #[clap(long, default_value = "3")]
+    nr_repeat_benchmarks: usize,
+    /// Do not run full benchmarks if no benchmark executable has changed
+    #[clap(long)]
+    check_executables_changed: bool,
+}

cli/benches/time_bench.rs

@@ -26,11 +26,11 @@ fn bench_group_wall_time(c: &mut Criterion) {
 
     let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![
         bench_nqueens,       // queens 11
-        bench_cfold,         // e = mkExpr 17 1
+        /*bench_cfold,         // e = mkExpr 17 1
         bench_deriv,         // nest deriv 8 f
         bench_rbtree_ck,     // ms = makeMap 5 80000
         bench_rbtree_delete, // m = makeMap 100000
-        bench_quicksort,     // list size 10000
+        bench_quicksort,*/     // list size 10000
     ];
 
     for bench_func in bench_funcs.iter() {

cli/cli_utils/src/bench_utils.rs

@@ -79,7 +79,7 @@ pub fn bench_nqueens<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup
         &example_file("benchmarks", "NQueens.roc"),
         "11",
         "nqueens",
-        "2680\n", //2680-14200
+        "14200\n", //2680-14200
         bench_group_opt,
     );
 }