Use Codspeed for continous benchmarking (#6896)

.github/workflows/benchmark.yaml

@@ -1,136 +0,0 @@
-name: Benchmark
-
-on:
-  pull_request:
-    paths:
-      - "Cargo.toml"
-      - "Cargo.lock"
-      - "rust-toolchain"
-      - "crates/**"
-      - "!crates/ruff_dev"
-      - "!crates/ruff_shrinking"
-
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: true
-
-jobs:
-  run-benchmark:
-    if: github.event_name == 'pull_request'
-    name: "Run | ${{ matrix.os }}"
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest]
-    runs-on: ${{ matrix.os }}
-
-    steps:
-      - name: "PR - Checkout Branch"
-        uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: "PR - Install Rust toolchain"
-        run: rustup show
-
-      - uses: Swatinem/rust-cache@v2
-
-      - name: "PR - Build benchmarks"
-        run: cargo bench -p ruff_benchmark --no-run
-
-      - name: "PR - Run benchmarks"
-        run: cargo benchmark --save-baseline=pr
-
-      - name: "Main - Checkout Branch"
-        uses: actions/checkout@v3
-        with:
-          clean: false
-          ref: main
-
-      - name: "Main - Install Rust toolchain"
-        run: rustup show
-
-      - name: "Main - Build benchmarks"
-        run: cargo bench -p ruff_benchmark --no-run
-
-      - name: "Main - Run benchmarks"
-        run: cargo benchmark --save-baseline=main
-
-      - name: "Upload benchmark results"
-        uses: actions/upload-artifact@v3
-        with:
-          name: benchmark-results-${{ matrix.os }}
-          path: ./target/criterion
-
-      # Cleanup
-      - name: Remove Criterion Artifact
-        uses: JesseTG/rm@v1.0.3
-        with:
-          path: ./target/criterion
-
-  benchmark-compare:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    name: Compare
-    needs:
-      - run-benchmark
-
-    steps:
-      - name: "Install Rust toolchain"
-        run: rustup show
-
-      - name: "Install critcmp"
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-
-      - name: "Linux | Download PR benchmark results"
-        uses: actions/download-artifact@v3
-        with:
-          name: benchmark-results-ubuntu-latest
-          path: ./target/criterion
-
-      - name: "Linux | Compare benchmark results"
-        shell: bash
-        run: |
-          echo "### Benchmark" >> summary.md
-          echo "#### Linux" >> summary.md
-          echo "\`\`\`" >> summary.md
-          critcmp main pr >> summary.md
-          echo "\`\`\`" >> summary.md
-          echo "" >> summary.md
-
-      - name: "Linux | Cleanup benchmark results"
-        run: rm -rf ./target/criterion
-
-      - name: "Windows | Download PR benchmark results"
-        uses: actions/download-artifact@v3
-        with:
-          name: benchmark-results-windows-latest
-          path: ./target/criterion
-
-      - name: "Windows | Compare benchmark results"
-        shell: bash
-        run: |
-          echo "#### Windows" >> summary.md
-          echo "\`\`\`" >> summary.md
-          critcmp main pr >> summary.md
-          echo "\`\`\`" >> summary.md
-          echo "" >> summary.md
-
-          echo ${{ github.event.pull_request.number }} > pr-number
-
-          cat summary.md > $GITHUB_STEP_SUMMARY
-
-      - uses: actions/upload-artifact@v3
-        name: Upload PR Number
-        with:
-          name: pr-number
-          path: pr-number
-
-      - uses: actions/upload-artifact@v3
-        name: Upload Summary
-        with:
-          name: summary
-          path: summary.md

.github/workflows/ci.yaml

@@ -341,3 +341,28 @@ jobs:
         run: cat target/progress_projects_stats.txt > $GITHUB_STEP_SUMMARY
       - name: "Remove checkouts from cache"
         run: rm -r target/progress_projects
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v3
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: "Build benchmarks"
+        run: cargo codspeed build --features codspeed -p ruff_benchmark
+
+      - name: "Run benchmarks"
+        uses: CodSpeedHQ/action@v1
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}

.github/workflows/pr-comment.yaml

@@ -2,7 +2,7 @@ name: PR Check Comment
 
 on:
   workflow_run:
-    workflows: [CI, Benchmark]
+    workflows: [CI]
     types: [completed]
   workflow_dispatch:
     inputs:
@@ -43,35 +43,16 @@ jobs:
           path: pr/ecosystem
           if_no_artifact_found: ignore
 
-      - uses: dawidd6/action-download-artifact@v2
-        name: "Download Benchmark Result"
-        id: download-benchmark-result
-        if: steps.pr-number.outputs.pr-number
-        with:
-          name: summary
-          workflow: benchmark.yaml
-          pr: ${{ steps.pr-number.outputs.pr-number }}
-          path: pr/benchmark
-          if_no_artifact_found: ignore
-
       - name: Generate Comment
         id: generate-comment
-        if: steps.download-ecosystem-result.outputs.found_artifact == 'true' || steps.download-benchmark-result.outputs.found_artifact == 'true'
+        if: steps.download-ecosystem-result.outputs.found_artifact == 'true'
         run: |
           echo 'comment<<EOF' >> $GITHUB_OUTPUT
           echo '## PR Check Results' >> $GITHUB_OUTPUT
 
-          if [[ -f pr/ecosystem/ecosystem-result ]]
-          then
-            echo "### Ecosystem" >> $GITHUB_OUTPUT
-            cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
-            echo "" >> $GITHUB_OUTPUT
-          fi
-
-          if [[ -f pr/benchmark/summary.md ]]
-          then
-            cat pr/benchmark/summary.md >> $GITHUB_OUTPUT
-          fi
+          echo "### Ecosystem" >> $GITHUB_OUTPUT
+          cat pr/ecosystem/ecosystem-result >> $GITHUB_OUTPUT
+          echo "" >> $GITHUB_OUTPUT
 
           echo 'EOF' >> $GITHUB_OUTPUT
 

Cargo.lock

@@ -414,6 +414,28 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "codspeed"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeec2fbed4969dc38b5ca201115dd5c2614b8ef78e0a7221dd5f0977fb1552b"
+dependencies = [
+ "colored",
+ "libc",
+ "serde_json",
+]
+
+[[package]]
+name = "codspeed-criterion-compat"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b13f0a08d40ce7c95bdf288f725b975e62fcadfa8ba152340943bab6de43af7"
+dependencies = [
+ "codspeed",
+ "colored",
+ "criterion",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.0"
@@ -2138,6 +2160,7 @@ dependencies = [
 name = "ruff_benchmark"
 version = "0.0.0"
 dependencies = [
+ "codspeed-criterion-compat",
  "criterion",
  "mimalloc",
  "once_cell",
@@ -2696,9 +2719,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.100"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
+checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
 dependencies = [
  "itoa",
  "ryu",

crates/ruff_benchmark/Cargo.toml

@@ -32,6 +32,8 @@ serde.workspace = true
 serde_json.workspace = true
 url = "2.3.1"
 ureq = "2.6.2"
+criterion = { version = "0.5.1"}
+codspeed-criterion-compat = { version="2.1.0", optional = true}
 
 [dev-dependencies]
 ruff.path = "../ruff"
@@ -39,7 +41,9 @@ ruff_python_ast.path = "../ruff_python_ast"
 ruff_python_formatter = { path = "../ruff_python_formatter" }
 ruff_python_index = { path = "../ruff_python_index" }
 ruff_python_parser = { path = "../ruff_python_parser" }
-criterion = { version = "0.5.1"}
+
+[features]
+codspeed = ["codspeed-criterion-compat"]
 
 [target.'cfg(target_os = "windows")'.dev-dependencies]
 mimalloc = "0.1.34"

crates/ruff_benchmark/benches/formatter.rs

@@ -1,8 +1,9 @@
 use std::path::Path;
 use std::time::Duration;
 
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
-
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_formatter::{format_node, PyFormatOptions};
 use ruff_python_index::CommentRangesBuilder;

crates/ruff_benchmark/benches/linter.rs

@@ -1,14 +1,12 @@
 use std::time::Duration;
 
-use criterion::measurement::WallTime;
-use criterion::{
-    criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
-};
-
 use ruff::linter::lint_only;
 use ruff::settings::{flags, Settings};
 use ruff::source_kind::SourceKind;
 use ruff::RuleSelector;
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_ast::PySourceType;
 
@@ -43,7 +41,7 @@ fn create_test_cases() -> Result<Vec<TestCase>, TestFileDownloadError> {
     ])
 }
 
-fn benchmark_linter(mut group: BenchmarkGroup<WallTime>, settings: &Settings) {
+fn benchmark_linter(mut group: BenchmarkGroup, settings: &Settings) {
     let test_cases = create_test_cases().unwrap();
 
     for case in test_cases {

crates/ruff_benchmark/benches/parser.rs

@@ -1,7 +1,8 @@
 use std::time::Duration;
 
-use criterion::measurement::WallTime;
-use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
+use ruff_benchmark::criterion::{
+    criterion_group, criterion_main, measurement::WallTime, BenchmarkId, Criterion, Throughput,
+};
 use ruff_benchmark::{TestCase, TestCaseSpeed, TestFile, TestFileDownloadError};
 use ruff_python_ast::statement_visitor::{walk_stmt, StatementVisitor};
 use ruff_python_ast::Stmt;

crates/ruff_benchmark/src/criterion.rs

@@ -0,0 +1,11 @@
+//! This module re-exports the criterion API but picks the right backend depending on whether
+//! the benchmarks are built to run locally or with codspeed
+
+#[cfg(not(codspeed))]
+pub use criterion::*;
+
+#[cfg(not(codspeed))]
+pub type BenchmarkGroup<'a> = criterion::BenchmarkGroup<'a, measurement::WallTime>;
+
+#[cfg(codspeed)]
+pub use codspeed_criterion_compat::*;

crates/ruff_benchmark/src/lib.rs

@@ -1,3 +1,5 @@
+pub mod criterion;
+
 use std::fmt::{Display, Formatter};
 use std::path::PathBuf;
 use std::process::Command;