Add formatter progress tracking to CI (#5919)

**Summary** Add a formatter progress testing script to CI. This script
will 1) print the black compability on each run 2) catch regressions wrt
to formatter stability, emitting invalid syntax and other kinds of bugs
(e.g. #5917) before they land on main 3) have an additional layer of
real world tests when implementing new nodes or other new formatter
code.

This is currently a bash script, i'm not sure if we want to keep it that
way, or switch to e.g. the regular ecosystem scripts. The output
separation of `format_dev` could also use some polishing. We should also
consider pinning commits so we don't get spurious regression when they
change their code.

**Test Plan** The script extends CI.
This commit is contained in:
konsti 2023-07-24 11:12:42 +02:00 committed by GitHub
parent dfa81b6fe0
commit 8a7dcb794b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 57 additions and 4 deletions

View file

@ -319,8 +319,8 @@ jobs:
if: ${{ env.MKDOCS_INSIDERS_SSH_KEY_EXISTS != 'true' }}
run: mkdocs build --strict -f mkdocs.generated.yml
check-formatter-stability:
name: "Check formatter stability"
check-formatter-ecosystem:
name: "Formatter ecosystem and progress checks"
runs-on: ubuntu-latest
needs: determine_changes
if: needs.determine_changes.outputs.formatter == 'true'
@ -330,7 +330,12 @@ jobs:
run: rustup show
- name: "Cache rust"
uses: Swatinem/rust-cache@v2
- name: "Formatter progress"
run: scripts/formatter_progress.sh
- name: "Github step summary"
run: grep "similarity index" target/progress_projects_report.txt | sort > $GITHUB_STEP_SUMMARY
# CPython is not black formatted, so we run only the stability check
- name: "Clone CPython 3.10"
run: git clone --branch 3.10 --depth 1 https://github.com/python/cpython.git crates/ruff/resources/test/cpython
- name: "Check stability"
- name: "Check CPython stability"
run: cargo run --bin ruff_dev -- format-dev --stability-check crates/ruff/resources/test/cpython

View file

@ -255,7 +255,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
bar.suspend(|| {
println!(
"Finished {} with {} files (similarity index {:.3}) in {:.2}s",
"Finished {}: {} files, similarity index {:.3}, {:.2}s",
project_path.display(),
result.file_count,
result.statistics.similarity_index(),
@ -275,6 +275,7 @@ fn format_dev_multi_project(args: &Args) -> anyhow::Result<bool> {
bar.inc(1);
}
}
bar.finish_and_clear();
}
let duration = start.elapsed();

47
scripts/formatter_progress.sh Executable file
View file

@ -0,0 +1,47 @@
#!/usr/bin/env bash
# The pinned revisions are the latest of this writing, update freely
set -ex
target=$(git rev-parse --show-toplevel)/target
dir="$target/progress_projects"
mkdir -p "$dir"
# small util library
if [ ! -d "$dir/build" ]; then
git clone --filter=tree:0 https://github.com/pypa/build "$dir/build"
git -C "$dir/build" checkout d90f9ac6503a40ddbfaef94b7a7040f87178a4b3
fi
# web framework that implements a lot of magic
if [ ! -d "$dir/django" ]; then
git clone --filter=tree:0 https://github.com/django/django "$dir/django"
git -C "$dir/django" checkout 95e4d6b81312fdd9f8ebf3385be1c1331168b5cf
fi
# an ML project
if [ ! -d "$dir/transformers" ]; then
git clone --filter=tree:0 https://github.com/huggingface/transformers "$dir/transformers"
git -C "$dir/transformers" checkout c9a82be592ca305180a7ab6a36e884bca1d426b8
fi
# type annotations
if [ ! -d "$dir/typeshed" ]; then
git clone --filter=tree:0 https://github.com/python/typeshed "$dir/typeshed"
git -C "$dir/typeshed" checkout 7d33060e6ae3ebe54462a891f0c566c97371915b
fi
# python 3.11, typing and 100% test coverage
if [ ! -d "$dir/warehouse" ]; then
git clone --filter=tree:0 https://github.com/pypi/warehouse "$dir/warehouse"
git -C "$dir/warehouse" checkout fe6455c0a946e81f61d72edc1049f536d8bba903
fi
# django project
if [ ! -d "$dir/zulip" ]; then
git clone --filter=tree:0 https://github.com/zulip/zulip "$dir/zulip"
git -C "$dir/zulip" checkout 6cb080c4479546a7f5cb017fcddea56605910b48
fi
# Uncomment if you want to update the hashes
# for i in "$dir"/*/; do git -C "$i" switch main && git -C "$i" pull && echo "# $(basename "$i") $(git -C "$i" rev-parse HEAD)"; done
time cargo run --bin ruff_dev -- format-dev --stability-check --error-file "$target/progress_projects_errors.txt" \
--multi-project "$dir" >"$target/progress_projects_report.txt"
grep "similarity index" "$target/progress_projects_report.txt" | sort