cleaned up benchmark script, now doing 200 runs per bench repeated 3 times to confirm

.gitignore

@@ -30,3 +30,6 @@ sccache_dir
 
 # self-contained benchmark folder
 bench-folder*
+
+# earthly
+earthly_log.txt

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "roc_std",
     "docs",
 ]
+exclude = ["ci/bench-runner"]
 # Needed to be able to run `cargo run -p roc_cli --no-default-features` -
 # see www/build.sh for more.
 #

ci/bench-runner.sh

@@ -1,84 +1,87 @@
 #!/usr/bin/env bash
 
 # script to return exit code 1 if benchmarks have regressed
+# script assumes we are in repo root
 
-# benchmark trunk
 ulimit -s unlimited
-cd bench-folder-trunk
-./target/release/deps/time_bench --bench
-cd ..
-
-# move benchmark results so they can be compared later
-cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
-
-cd bench-folder-branch
 
 LOG_FILE="bench_log.txt"
-touch $LOG_FILE
 
-FULL_CMD=" ./target/release/deps/time_bench --bench"
-echo $FULL_CMD
+NR_REPEATS=3
 
-script -efq $LOG_FILE -c "$FULL_CMD"
-EXIT_CODE=$?
+for ctr in `seq 1 $NR_REPEATS`;
+    do
+        #
+        # <run benchmarks>
+        #
+        # delete criterion folder to remove old benchmark data (ignore error if folder does not exist)
+        rm -rf bench-folder-trunk/target/criterion
+        rm -rf bench-folder-branch/target/criterion
         
-if cat $LOG_FILE | grep -q "regressed"; then
-
-    grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_1.txt
-    echo "regression(s) detected in:"
-    cat slow_benches_1.txt
-    echo ""
-    echo ""
-    echo "------<<<<<<>>>>>>------"
-    echo "Benchmark detected regression. Running benchmark again to confirm..."
-    echo "------<<<<<<>>>>>>------"
-    echo ""
-    echo ""
-
-    # delete criterion folder to remove old benchmark data
-    rm -rf ./target/criterion
-
-    # benchmark trunk again
-    cd ../bench-folder-trunk
-    rm -rf target/criterion
+        cd bench-folder-trunk
+        # benchmark trunk
         ./target/release/deps/time_bench --bench
+        cd ..
 
-    cd ../bench-folder-branch
-    cp -r ../bench-folder-trunk/target/criterion ./target
+        # move benchmark results so they can be compared later
+        cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
 
-    rm $LOG_FILE
+        cd bench-folder-branch
+
+        # ignore error if file does not exist
+        rm -f $LOG_FILE 2>&1
         touch $LOG_FILE
 
+        FULL_CMD=" ./target/release/deps/time_bench --bench"
+        echo $FULL_CMD
+
         script -efq $LOG_FILE -c "$FULL_CMD"
+
         EXIT_CODE=$?
+        #
+        # </run benchmarks>
+        #
 
-    if cat $LOG_FILE | grep -q "regressed"; then
+        #
+        # <save which tests regressed>
+        #
+        REGRESSED_TESTS_FILE_NAME="regressed_$ctr.txt"
         
-        grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_2.txt
-        echo "regression(s) detected in:"
-        cat slow_benches_2.txt
+        grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > $REGRESSED_TESTS_FILE_NAME
 
-        if [[ $(grep -Fxf slow_benches_1.txt slow_benches_2.txt | wc -l) -gt 0 ]]; then
+        #
+        # </save which tests regressed>
+        #
+
+        if [ $(cat $REGRESSED_TESTS_FILE_NAME | wc -l) -gt 0 ]; then 
+            if [ $ctr -ne $NR_REPEATS ]; then
+                echo ""
+                echo ""
+                echo "------<<<<<<>>>>>>------"
+                echo "Benchmark regression detected for:"
+                cat $REGRESSED_TESTS_FILE_NAME
+                echo "Running benchmarks again to confirm regression is real..."
+                echo "------<<<<<<>>>>>>------"
+                echo ""
+                echo ""
+            else
                 echo ""
                 echo ""
                 echo "------<<<<<<!!!!!!>>>>>>------"
-            echo "Benchmarks were run twice and a regression was detected both times for the following benchmarks:"
-            grep -Fxf slow_benches_1.txt slow_benches_2.txt
+                echo "Benchmarks were run $NR_REPEATS times and a regression was detected every time for the following benchmarks:"
+                cat regressed_*.txt > regressed.txt
+                sort regressed.txt | uniq -d 
                 echo "------<<<<<<!!!!!!>>>>>>------"
                 echo ""
                 echo ""
                 exit 1
-        else
-            echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke."
-            exit 0
             fi
         else
-        echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke."
-        exit 0
-    fi
-else
             echo ""
             echo "Benchmark execution finished with exit code: $EXIT_CODE."
             echo ""
             exit $EXIT_CODE
-fi
+        fi
+
+        cd ..
+    done  

cli/benches/time_bench.rs

@@ -7,9 +7,9 @@ use criterion::{
 
 fn bench_group_wall_time(c: &mut Criterion) {
     let mut group = c.benchmark_group("bench-group_wall-time");
-    // calculate statistics based on a fixed(flat) 300 runs
+    // calculate statistics based on a fixed(flat) 200 runs
     group.sampling_mode(SamplingMode::Flat);
-    group.sample_size(300);
+    group.sample_size(200);
 
     let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![
         bench_nqueens,       // queens 11

cli/cli_utils/src/bench_utils.rs

@@ -79,7 +79,7 @@ pub fn bench_nqueens<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup
         &example_file("benchmarks", "NQueens.roc"),
         "11",
         "nqueens",
-        "2680\n",
+        "2680\n",//2680-14200
         bench_group_opt,
     );
 }