cleaned up benchmark script, now doing 200 runs per bench repeated 3 times to confirm

This commit is contained in:
Anton-4 2021-07-20 14:10:35 +02:00
parent e7223fe176
commit a3f7c6accf
5 changed files with 74 additions and 67 deletions

3
.gitignore vendored
View file

@ -30,3 +30,6 @@ sccache_dir
# self-contained benchmark folder # self-contained benchmark folder
bench-folder* bench-folder*
# earthly
earthly_log.txt

View file

@ -32,6 +32,7 @@ members = [
"roc_std", "roc_std",
"docs", "docs",
] ]
exclude = ["ci/bench-runner"]
# Needed to be able to run `cargo run -p roc_cli --no-default-features` - # Needed to be able to run `cargo run -p roc_cli --no-default-features` -
# see www/build.sh for more. # see www/build.sh for more.
# #

View file

@ -1,84 +1,87 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# script to return exit code 1 if benchmarks have regressed # script to return exit code 1 if benchmarks have regressed
# script assumes we are in repo root
# benchmark trunk
ulimit -s unlimited ulimit -s unlimited
cd bench-folder-trunk
./target/release/deps/time_bench --bench
cd ..
# move benchmark results so they can be compared later
cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
cd bench-folder-branch
LOG_FILE="bench_log.txt" LOG_FILE="bench_log.txt"
touch $LOG_FILE
FULL_CMD=" ./target/release/deps/time_bench --bench" NR_REPEATS=3
echo $FULL_CMD
script -efq $LOG_FILE -c "$FULL_CMD" for ctr in `seq 1 $NR_REPEATS`;
EXIT_CODE=$? do
#
# <run benchmarks>
#
# delete criterion folder to remove old benchmark data (ignore error if folder does not exist)
rm -rf bench-folder-trunk/target/criterion
rm -rf bench-folder-branch/target/criterion
cd bench-folder-trunk
# benchmark trunk
./target/release/deps/time_bench --bench
cd ..
if cat $LOG_FILE | grep -q "regressed"; then # move benchmark results so they can be compared later
cp -r bench-folder-trunk/target/criterion bench-folder-branch/target/
grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_1.txt cd bench-folder-branch
echo "regression(s) detected in:"
cat slow_benches_1.txt
echo ""
echo ""
echo "------<<<<<<>>>>>>------"
echo "Benchmark detected regression. Running benchmark again to confirm..."
echo "------<<<<<<>>>>>>------"
echo ""
echo ""
# delete criterion folder to remove old benchmark data # ignore error if file does not exist
rm -rf ./target/criterion rm -f $LOG_FILE 2>&1
touch $LOG_FILE
# benchmark trunk again FULL_CMD=" ./target/release/deps/time_bench --bench"
cd ../bench-folder-trunk echo $FULL_CMD
rm -rf target/criterion
./target/release/deps/time_bench --bench
cd ../bench-folder-branch script -efq $LOG_FILE -c "$FULL_CMD"
cp -r ../bench-folder-trunk/target/criterion ./target
rm $LOG_FILE EXIT_CODE=$?
touch $LOG_FILE #
# </run benchmarks>
#
script -efq $LOG_FILE -c "$FULL_CMD" #
EXIT_CODE=$? # <save which tests regressed>
#
REGRESSED_TESTS_FILE_NAME="regressed_$ctr.txt"
grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > $REGRESSED_TESTS_FILE_NAME
if cat $LOG_FILE | grep -q "regressed"; then #
# </save which tests regressed>
#
grep -B3 "regressed" $LOG_FILE | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g' | grep -o "\".*\"" | rev | cut -d' ' -f1 | rev > slow_benches_2.txt if [ $(cat $REGRESSED_TESTS_FILE_NAME | wc -l) -gt 0 ]; then
echo "regression(s) detected in:" if [ $ctr -ne $NR_REPEATS ]; then
cat slow_benches_2.txt echo ""
echo ""
if [[ $(grep -Fxf slow_benches_1.txt slow_benches_2.txt | wc -l) -gt 0 ]]; then echo "------<<<<<<>>>>>>------"
echo "" echo "Benchmark regression detected for:"
echo "" cat $REGRESSED_TESTS_FILE_NAME
echo "------<<<<<<!!!!!!>>>>>>------" echo "Running benchmarks again to confirm regression is real..."
echo "Benchmarks were run twice and a regression was detected both times for the following benchmarks:" echo "------<<<<<<>>>>>>------"
grep -Fxf slow_benches_1.txt slow_benches_2.txt echo ""
echo "------<<<<<<!!!!!!>>>>>>------" echo ""
echo "" else
echo "" echo ""
exit 1 echo ""
echo "------<<<<<<!!!!!!>>>>>>------"
echo "Benchmarks were run $NR_REPEATS times and a regression was detected every time for the following benchmarks:"
cat regressed_*.txt > regressed.txt
sort regressed.txt | uniq -d
echo "------<<<<<<!!!!!!>>>>>>------"
echo ""
echo ""
exit 1
fi
else else
echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke." echo ""
exit 0 echo "Benchmark execution finished with exit code: $EXIT_CODE."
echo ""
exit $EXIT_CODE
fi fi
else
echo "Benchmarks were run twice and a regression was detected on one run. We assume this was a fluke." cd ..
exit 0 done
fi
else
echo ""
echo "Benchmark execution finished with exit code: $EXIT_CODE."
echo ""
exit $EXIT_CODE
fi

View file

@ -7,9 +7,9 @@ use criterion::{
fn bench_group_wall_time(c: &mut Criterion) { fn bench_group_wall_time(c: &mut Criterion) {
let mut group = c.benchmark_group("bench-group_wall-time"); let mut group = c.benchmark_group("bench-group_wall-time");
// calculate statistics based on a fixed(flat) 300 runs // calculate statistics based on a fixed(flat) 200 runs
group.sampling_mode(SamplingMode::Flat); group.sampling_mode(SamplingMode::Flat);
group.sample_size(300); group.sample_size(200);
let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![ let bench_funcs: Vec<fn(Option<&mut BenchmarkGroup<WallTime>>) -> ()> = vec![
bench_nqueens, // queens 11 bench_nqueens, // queens 11

View file

@ -79,7 +79,7 @@ pub fn bench_nqueens<T: Measurement>(bench_group_opt: Option<&mut BenchmarkGroup
&example_file("benchmarks", "NQueens.roc"), &example_file("benchmarks", "NQueens.roc"),
"11", "11",
"nqueens", "nqueens",
"2680\n", "2680\n",//2680-14200
bench_group_opt, bench_group_opt,
); );
} }