From 35b0c4d422ffce699cf076a65714cab5f55fd85a Mon Sep 17 00:00:00 2001 From: Andrew Gallant Date: Thu, 6 Mar 2025 15:31:40 -0500 Subject: [PATCH] ci: break `./scripts/test` into pieces... ... so that we can run each piece in its own job in CI. This creates an obscene number of jobs, but I'm really hoping this cuts down on the total wall clock time. --- .github/workflows/ci.yml | 268 ++++++++++++++++++++-------- scripts/test | 78 ++------ scripts/test-all | 10 ++ scripts/test-default | 10 ++ scripts/test-only-bundle | 17 ++ scripts/test-only-core | 35 ++++ scripts/test-various-feature-combos | 26 +++ 7 files changed, 300 insertions(+), 144 deletions(-) create mode 100755 scripts/test-all create mode 100755 scripts/test-default create mode 100755 scripts/test-only-bundle create mode 100755 scripts/test-only-core create mode 100755 scripts/test-various-feature-combos diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd1a670..b5ab935 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,10 +28,11 @@ permissions: contents: read jobs: - # This runs tests under a number of different feature combinations. Jiff has - # too many features to do the full powerset, but we capture a number of - # configurations here. - testfull: + # The standard `cargo test --all` on many platforms. + # + # For the other `test-*` jobs, we cut this down to Linux/nightly and + # macOS/nightly, just to avoid more jobs than we probably need. + test-default: runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -65,7 +66,190 @@ jobs: run: sed -i.bak 's/^edition = "2021"$/edition = "2024"/' Cargo.toml - name: Bump rust-version to Rust 1.85 run: sed -i.bak 's/^rust-version = "1\.70"$/rust-version = "1.85"/' Cargo.toml - - run: ./scripts/test + - run: ./scripts/test-default + + # Test all features enabled. + test-all: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: nightly + os: ubuntu-latest + rust: nightly + - build: macos + os: macos-latest + rust: nightly + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + # In Rust 2024, doc tests are (mostly) all compiled into a single + # executable and then run once. This is a lot faster for Jiff, which + # has 1000+ doc tests. Since this job is where we run most of our + # tests (and we run MSRV on a different job), we just bump to Rust 2024 + # to take advantage of this. + - name: Switch to Rust 2024 + run: sed -i.bak 's/^edition = "2021"$/edition = "2024"/' Cargo.toml + - name: Bump rust-version to Rust 1.85 + run: sed -i.bak 's/^rust-version = "1\.70"$/rust-version = "1.85"/' Cargo.toml + - run: ./scripts/test-all + + # Test Jiff when only the bundled tzdb is enabled. + test-only-bundle: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: nightly + os: ubuntu-latest + rust: nightly + - build: macos + os: macos-latest + rust: nightly + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + # In Rust 2024, doc tests are (mostly) all compiled into a single + # executable and then run once. This is a lot faster for Jiff, which + # has 1000+ doc tests. Since this job is where we run most of our + # tests (and we run MSRV on a different job), we just bump to Rust 2024 + # to take advantage of this. + - name: Switch to Rust 2024 + run: sed -i.bak 's/^edition = "2021"$/edition = "2024"/' Cargo.toml + - name: Bump rust-version to Rust 1.85 + run: sed -i.bak 's/^rust-version = "1\.70"$/rust-version = "1.85"/' Cargo.toml + - run: ./scripts/test-only-bundle + + # Test Jiff in core-only (including no-alloc) configurations. + test-core: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: nightly + os: ubuntu-latest + rust: nightly + - build: macos + os: macos-latest + rust: nightly + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - run: ./scripts/test-only-core + + # Test a number of different feature combinations. + # + # Jiff has too many features to do the full powerset, so this just tries to + # get decent coverage over what I perceive to be likely configs. + test-various-feature-combos: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - build: nightly + os: ubuntu-latest + rust: nightly + - build: macos + os: macos-latest + rust: nightly + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + - run: ./scripts/test-various-feature-combos + + # Run tests with `testrelease` profile. + # + # This is useful because Jiff's ranged integers internally track min/max + # values in debug mode, but don't in release mode. So there may occasionally + # be a discrepancy that is worth testing here. + test-release: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + # In Rust 2024, doc tests are (mostly) all compiled into a single + # executable and then run once. This is a lot faster for Jiff, which has + # 1000+ doc tests. Since we run doc tests in this job, we just bump to Rust + # 2024 to take advantage of this. + - name: Switch to Rust 2024 + run: sed -i.bak 's/^edition = "2021"$/edition = "2024"/' Cargo.toml + - name: Bump rust-version to Rust 1.85 + run: sed -i.bak 's/^rust-version = "1\.70"$/rust-version = "1.85"/' Cargo.toml + - name: Run tests under release mode + run: cargo test --verbose --all --profile testrelease + + # Test that documentation gets built. + test-doc: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Build docs with all features enabled + run: cargo doc --verbose --all-features + + # Run benchmarks as tests. + test-bench: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Run benchmark tests + run: | + cargo bench --manifest-path bench/Cargo.toml -- --test + + # Runs miri on a subset of Jiff's test suite. This doesn't quite cover + # everything. In particular, `miri` and `insta` cannot play nice together, + # and `insta` is used a lot among Jiff's tests. However, the primary reason + # why we want to run `miri` (at present, 2025-02-25) is to check out pointer + # tagging representation of `jiff::tz::TimeZone`. And we can run the tests in + # `tz`. + test-miri: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + # We use nightly here so that we can use miri I guess? + toolchain: nightly + components: miri + - name: "Run tests on `jiff::tz` under miri" + run: "cargo miri test tz:: --verbose" + - name: Run proc macro integration tests under miri + run: cargo miri test --test integration procmacro --verbose # Test for Windows. We test fewer configurations here because it just # takes forever otherwise. @@ -321,80 +505,6 @@ jobs: cd crates/jiff-wasm wasm-pack test --node - # Run tests with `testrelease` profile. - # - # This is useful because Jiff's ranged integers internally track min/max - # values in debug mode, but don't in release mode. So there may occasionally - # be a discrepancy that is worth testing here. - testrelease: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - # In Rust 2024, doc tests are (mostly) all compiled into a single - # executable and then run once. This is a lot faster for Jiff, which has - # 1000+ doc tests. Since we run doc tests in this job, we just bump to Rust - # 2024 to take advantage of this. - - name: Switch to Rust 2024 - run: sed -i.bak 's/^edition = "2021"$/edition = "2024"/' Cargo.toml - - name: Bump rust-version to Rust 1.85 - run: sed -i.bak 's/^rust-version = "1\.70"$/rust-version = "1.85"/' Cargo.toml - - name: Run tests under release mode - run: cargo test --verbose --all --profile testrelease - - # Test that documentation gets built. - testdoc: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - name: Build docs with all features enabled - run: cargo doc --verbose --all-features - - # Run benchmarks as tests. - testbench: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - name: Run benchmark tests - run: | - cargo bench --manifest-path bench/Cargo.toml -- --test - - # Runs miri on a subset of Jiff's test suite. This doesn't quite cover - # everything. In particular, `miri` and `insta` cannot play nice together, - # and `insta` is used a lot among Jiff's tests. However, the primary reason - # why we want to run `miri` (at present, 2025-02-25) is to check out pointer - # tagging representation of `jiff::tz::TimeZone`. And we can run the tests in - # `tz`. - miri: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - # We use nightly here so that we can use miri I guess? - toolchain: nightly - components: miri - - name: "Run tests on `jiff::tz` under miri" - run: "cargo miri test tz:: --verbose" - - name: Run proc macro integration tests under miri - run: cargo miri test --test integration procmacro --verbose - # Check that all files are formatted properly. rustfmt: runs-on: ubuntu-latest diff --git a/scripts/test b/scripts/test index e5ca1d9..ffa3a60 100755 --- a/scripts/test +++ b/scripts/test @@ -1,5 +1,9 @@ #!/bin/bash +# This is a convenience script for running a broad swath of tests across +# features. We don't test the complete space, since the complete space is quite +# large. +# # This script is used to run a more exhaustive set of tests than what just # `cargo test` or even `cargo test --all` will do. Specifically, we run tests # under a number of different feature configurations. @@ -13,68 +17,12 @@ set -e # to pass --manifest-path to every `cargo` command. cd "$(dirname "$0")/.." -# This is a convenience script for running a broad swath of tests across -# features. We don't test the complete space, since the complete space is quite -# large. -echo "===== DEFAULT FEATURES =====" -cargo test --all - -echo "===== DEFAULT FEATURES WITH STATIC =====" -cargo test --features static - -# This one is useful because sometimes the bundled time zone database can -# behave differently than the system time zone database depending on the -# inputs. For example, the bundled database uses as few transitions as possible -# (this is tzdb's "slim" data model) and thus relies more heavily on POSIX -# time zone strings. So if there's a bug in POSIX time zone handling, you're -# likely to see it with the bundled database while missing it completely with -# the system database. -echo "===== WITH ONLY THE BUNDLED TIME ZONE DATABASE =====" -cargo test --no-default-features --features std,tz-system,tzdb-bundle-always - -# We test core-only mode specially. Sadly, in core-only mode, error messages -# are quite a bit worse. And this wreaks havoc with Jiff's snapshot tests on -# error messages. We could `cfg` all of them, but that's a huge pain and it's -# not clear that it's worth it. -# -# Since we use snapshot tests for more than error messages, this technique also -# risks accidentally passing a test that doesn't involve error messages. Sigh. -# We accept this risk because this still runs a lot of tests, and the tests -# that aren't covered are run in other configurations. Still, it's not ideal. -echo "===== CORE ONLY =====" -cargo build --no-default-features -INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --lib --no-default-features -INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --test integration --no-default-features - -# More core-only tests, when combined with compatible features. -features=( - "serde" - "logging" - "serde logging" - "static" -) -for f in "${features[@]}"; do - echo "===== COREONLY PLUS '$f' =====" - cargo build --no-default-features --features "$f" - INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --lib --no-default-features --features "$f" - INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --test integration --no-default-features --features "$f" -done - -features=( - "alloc" - "alloc tzdb-bundle-always" - "alloc serde" - "std" - "std tzdb-bundle-platform tzdb-bundle-always" - "std tzdb-bundle-platform tzdb-bundle-always tzdb-zoneinfo" - "std tzdb-bundle-platform tzdb-zoneinfo" - "std tzdb-bundle-always tzdb-zoneinfo" - "std tzdb-bundle-always logging" - "std tzdb-bundle-always serde" -) -for f in "${features[@]}"; do - echo "===== FEATURES: '$f' =====" - cargo build --no-default-features --features "$f" - cargo test --lib --no-default-features --features "$f" - cargo test --test integration --no-default-features --features "$f" -done +# This is split into separate scripts so that each one can be run in an +# independent job on CI. This speeds up CI. But locally, developer workstations +# are powerful enough that just running everything at once doesn't take that +# long (a few minutes on my aging i9-12900K). +./scripts/test-default +./scripts/test-all +./scripts/test-only-bundle +./scripts/test-only-core +./scripts/test-various-feature-combos diff --git a/scripts/test-all b/scripts/test-all new file mode 100755 index 0000000..2dfe72e --- /dev/null +++ b/scripts/test-all @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# cd to the directory containing this crate's Cargo.toml so that we don't need +# to pass --manifest-path to every `cargo` command. +cd "$(dirname "$0")/.." + +echo "===== ALL FEATURES =====" +cargo test --all-features diff --git a/scripts/test-default b/scripts/test-default new file mode 100755 index 0000000..8d0a89b --- /dev/null +++ b/scripts/test-default @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# cd to the directory containing this crate's Cargo.toml so that we don't need +# to pass --manifest-path to every `cargo` command. +cd "$(dirname "$0")/.." + +echo "===== DEFAULT FEATURES =====" +cargo test --all diff --git a/scripts/test-only-bundle b/scripts/test-only-bundle new file mode 100755 index 0000000..51ecb0b --- /dev/null +++ b/scripts/test-only-bundle @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +# cd to the directory containing this crate's Cargo.toml so that we don't need +# to pass --manifest-path to every `cargo` command. +cd "$(dirname "$0")/.." + +# This one is useful because sometimes the bundled time zone database can +# behave differently than the system time zone database depending on the +# inputs. For example, the bundled database uses as few transitions as possible +# (this is tzdb's "slim" data model) and thus relies more heavily on POSIX +# time zone strings. So if there's a bug in POSIX time zone handling, you're +# likely to see it with the bundled database while missing it completely with +# the system database. +echo "===== WITH ONLY THE BUNDLED TIME ZONE DATABASE =====" +cargo test --no-default-features --features std,tz-system,tzdb-bundle-always diff --git a/scripts/test-only-core b/scripts/test-only-core new file mode 100755 index 0000000..35931d8 --- /dev/null +++ b/scripts/test-only-core @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +# cd to the directory containing this crate's Cargo.toml so that we don't need +# to pass --manifest-path to every `cargo` command. +cd "$(dirname "$0")/.." + +# We test core-only mode specially. Sadly, in core-only mode, error messages +# are quite a bit worse. And this wreaks havoc with Jiff's snapshot tests on +# error messages. We could `cfg` all of them, but that's a huge pain and it's +# not clear that it's worth it. +# +# Since we use snapshot tests for more than error messages, this technique also +# risks accidentally passing a test that doesn't involve error messages. Sigh. +# We accept this risk because this still runs a lot of tests, and the tests +# that aren't covered are run in other configurations. Still, it's not ideal. +echo "===== CORE ONLY =====" +cargo build --no-default-features +INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --lib --no-default-features +INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --test integration --no-default-features + +# More core-only tests, when combined with compatible features. +features=( + "serde" + "logging" + "serde logging" + "static" +) +for f in "${features[@]}"; do + echo "===== COREONLY PLUS '$f' =====" + cargo build --no-default-features --features "$f" + INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --lib --no-default-features --features "$f" + INSTA_FORCE_PASS=1 INSTA_UPDATE=no cargo test --test integration --no-default-features --features "$f" +done diff --git a/scripts/test-various-feature-combos b/scripts/test-various-feature-combos new file mode 100755 index 0000000..e4081d2 --- /dev/null +++ b/scripts/test-various-feature-combos @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# cd to the directory containing this crate's Cargo.toml so that we don't need +# to pass --manifest-path to every `cargo` command. +cd "$(dirname "$0")/.." + +features=( + "alloc" + "alloc tzdb-bundle-always" + "alloc serde" + "std" + "std tzdb-bundle-platform tzdb-bundle-always" + "std tzdb-bundle-platform tzdb-bundle-always tzdb-zoneinfo" + "std tzdb-bundle-platform tzdb-zoneinfo" + "std tzdb-bundle-always tzdb-zoneinfo" + "std tzdb-bundle-always logging" + "std tzdb-bundle-always serde" +) +for f in "${features[@]}"; do + echo "===== FEATURES: '$f' =====" + cargo build --no-default-features --features "$f" + cargo test --lib --no-default-features --features "$f" + cargo test --test integration --no-default-features --features "$f" +done