cleaned up benchmark script, now doing 200 runs per bench repeated 3 times to confirm

.gitignore

@@ -30,3 +30,6 @@ sccache_dir
 
 # self-contained benchmark folder
 bench-folder*
+
+# earthly
+earthly_log.txt

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "roc_std",
     "docs",
 ]
+exclude = ["ci/bench-runner"]
 # Needed to be able to run `cargo run -p roc_cli --no-default-features` -
 # see www/build.sh for more.
 #

ci/bench-runner.sh

@@ -1,84 +1,87 @@
 #!/usr/bin/env bash
 
 # script to return exit code 1 if benchmarks have regressed
+# script assumes we are in repo root
 
-# benchmark trunk
 ulimit -s unlimited
-cd bench-folder-trunk
-./target/release/deps/time_bench --bench
-cd ..
-
-# move benchmark results so they can be compared later
-cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
-
-cd bench-folder-branch
 
 LOG_FILE="bench_log.txt"
-touch $LOG_FILE
 
-FULL_CMD=" ./target/release/deps/time_bench --bench"
+NR_REPEATS=3
-echo $FULL_CMD
 
-script -efq $LOG_FILE -c "$FULL_CMD"
+for ctr in `seq 1 $NR_REPEATS`;
-EXIT_CODE=$?
+    do
+        #
+        # <run benchmarks>
+        #
+        # delete criterion folder to remove old benchmark data (ignore error if folder does not exist)
+        rm -rf bench-folder-trunk/target/criterion
+        rm -rf bench-folder-branch/target/criterion
+        
+        cd bench-folder-trunk
+        # benchmark trunk
+        ./target/release/deps/time_bench --bench
+        cd ..
 
-if cat $LOG_FILE | grep -q "regressed"; then
+        # move benchmark results so they can be compared later
+        cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
 
-    grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_1.txt
+        cd bench-folder-branch
-    echo "regression(s) detected in:"
-    cat slow_benches_1.txt
-    echo ""
-    echo ""
-    echo "------<<<<<<>>>>>>------"
-    echo "Benchmark detected regression. Running benchmark again to confirm..."
-    echo "------<<<<<<>>>>>>------"
-    echo ""
-    echo ""
 
-    # delete criterion folder to remove old benchmark data
+        # ignore error if file does not exist
-    rm -rf ./target/criterion
+        rm -f $LOG_FILE 2>&1
+        touch $LOG_FILE
 
-    # benchmark trunk again
+        FULL_CMD=" ./target/release/deps/time_bench --bench"
-    cd ../bench-folder-trunk
+        echo $FULL_CMD
-    rm -rf target/criterion
-    ./target/release/deps/time_bench --bench
 
-    cd ../bench-folder-branch
+        script -efq $LOG_FILE -c "$FULL_CMD"
-    cp -r ../bench-folder-trunk/target/criterion ./target
 
-    rm $LOG_FILE
+        EXIT_CODE=$?
-    touch $LOG_FILE
+        #
+        # </run benchmarks>
+        #
 
-    script -efq $LOG_FILE -c "$FULL_CMD"
+        #
-    EXIT_CODE=$?
+        # <save which tests regressed>
+        #
+        REGRESSED_TESTS_FILE_NAME="regressed_$ctr.txt"
+        
+        grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > $REGRESSED_TESTS_FILE_NAME
 
-    if cat $LOG_FILE | grep -q "regressed"; then
+        #
+        # </save which tests regressed>
+        #
 
-        grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_2.txt
+        if [ $(cat $REGRESSED_TESTS_FILE_NAME | wc -l) -gt 0 ]; then 
-        echo "regression(s) detected in:"
+            if [ $ctr -ne $NR_REPEATS ]; then
-        cat slow_benches_2.txt
+                echo ""
-
+                echo ""
-        if [[ $(grep -Fxf slow_benches_1.txt slow_benches_2.txt | wc -l) -gt 0 ]]; then
+                echo "------<<<<<<>>>>>>------"
-            echo ""
+                echo "Benchmark regression detected for:"
-            echo ""
+                cat $REGRESSED_TESTS_FILE_NAME
-            echo "------<<<<<<!!!!!!>>>>>>------"
+                echo "Running benchmarks again to confirm regression is real..."
-            echo "Benchmarks were run twice and a regression was detected both times for the following benchmarks:"
+                echo "------<<<<<<>>>>>>------"
-            grep -Fxf slow_benches_1.txt slow_benches_2.txt
+                echo ""
-            echo "------<<<<<<!!!!!!>>>>>>------"
+                echo ""
-            echo ""
+            else
-            echo ""
+                echo ""
-            exit 1
+                echo ""
+                echo "------<<<<<<!!!!!!>>>>>>------"
+                echo "Benchmarks were run $NR_REPEATS times and a regression was detected every time for the following benchmarks:"
+                cat regressed_*.txt > regressed.txt
+                sort regressed.txt | uniq -d 
+                echo "------<<<<<<!!!!!!>>>>>>------"
+                echo ""
+                echo ""
+                exit 1
+            fi
         else
-            echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke."
+            echo ""
-            exit 0
+            echo "Benchmark execution finished with exit code: $EXIT_CODE."
+            echo ""
+            exit $EXIT_CODE
         fi
-    else
+
-        echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke."
+        cd ..
-        exit 0
+    done  
-    fi
-else
-    echo ""
-    echo "Benchmark execution finished with exit code: $EXIT_CODE."
-    echo ""
-    exit $EXIT_CODE
-fi

cli/benches/time_bench.rs

@@ -7,9 +7,9 @@ use criterion::{
 
 fn bench_group_wall_time(c: &mut Criterion) {
     let mut group = c.benchmark_group("bench-group_wall-time");
-    // calculate statistics based on a fixed(flat) 300 runs
+    // calculate statistics based on a fixed(flat) 200 runs
     group.sampling_mode(SamplingMode::Flat);
-    group.sample_size(300);
+    group.sample_size(200);
 
     let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![
         bench_nqueens,       // queens 11

cli/cli_utils/src/bench_utils.rs

@@ -79,7 +79,7 @@ pub fn bench_nqueens<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup
         &example_file("benchmarks", "NQueens.roc"),
         "11",
         "nqueens",
-        "2680\n",
+        "2680\n",//2680-14200
         bench_group_opt,
     );
 }