Run resolve/install benchmarks in ci (#3281)

## Summary

Runs resolver benchmarks in CI with CodSpeed.

.github/workflows/ci.yml

@@ -18,6 +18,26 @@ env:
   PYTHON_VERSION: "3.12"
 
 jobs:
+  determine_changes:
+    name: "Determine changes"
+    runs-on: ubuntu-latest
+    outputs:
+      # Flag that is raised when any code is changed
+      code: ${{ steps.changed.outputs.code_any_changed }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: tj-actions/changed-files@v44
+        id: changed
+        with:
+          files_yaml: |
+            code:
+              - "**/*"
+              - "!**/*.md"
+              - "!bin/**"
+              - "!assets/**"
   cargo-fmt:
     name: "cargo fmt"
     runs-on: ubuntu-latest
@@ -858,3 +878,31 @@ jobs:
 
       - name: "Validate embedded Python install"
         run: python ./scripts/check_embedded_python.py --uv ./uv.exe
+
+  benchmarks:
+    runs-on: ubuntu-latest
+    needs: determine_changes
+    if: ${{ github.repository == 'astral-sh/uv' && (needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main') }}
+    timeout-minutes: 20
+    steps:
+      - name: "Checkout Branch"
+        uses: actions/checkout@v4
+
+      - name: "Install Rust toolchain"
+        run: rustup show
+
+      - name: "Install codspeed"
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: "Build benchmarks"
+        run: cargo codspeed build --features codspeed -p bench
+
+      - name: "Run benchmarks"
+        uses: CodSpeedHQ/action@v2
+        with:
+          run: cargo codspeed run
+          token: ${{ secrets.CODSPEED_TOKEN }}

Cargo.lock

@@ -394,9 +394,23 @@ checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51"
 name = "bench"
 version = "0.0.0"
 dependencies = [
+ "anyhow",
+ "codspeed-criterion-compat",
  "criterion",
  "distribution-filename",
+ "distribution-types",
+ "fs-err",
+ "once_cell",
+ "pep508_rs",
  "platform-tags",
+ "tempfile",
+ "tokio",
+ "uv-cache",
+ "uv-client",
+ "uv-configuration",
+ "uv-interpreter",
+ "uv-resolver",
+ "uv-types",
 ]
 
 [[package]]
@@ -758,6 +772,28 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "codspeed"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089"
+dependencies = [
+ "colored",
+ "libc",
+ "serde_json",
+]
+
+[[package]]
+name = "codspeed-criterion-compat"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe"
+dependencies = [
+ "codspeed",
+ "colored",
+ "criterion",
+]
+
 [[package]]
 name = "color_quant"
 version = "1.1.0"
@@ -770,6 +806,16 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
 
+[[package]]
+name = "colored"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
+dependencies = [
+ "lazy_static",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "concurrent-queue"
 version = "2.4.0"
@@ -843,6 +889,7 @@ dependencies = [
  "ciborium",
  "clap",
  "criterion-plot",
+ "futures",
  "is-terminal",
  "itertools 0.10.5",
  "num-traits",
@@ -853,6 +900,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "tinytemplate",
+ "tokio",
  "walkdir",
 ]
 

crates/bench/Cargo.toml

@@ -22,8 +22,29 @@ name = "distribution-filename"
 path = "benches/distribution_filename.rs"
 harness = false
 
+[[bench]]
+name = "uv"
+path = "benches/uv.rs"
+harness = false
+
 [dependencies]
 distribution-filename = { workspace = true }
+distribution-types = { workspace = true }
+criterion = { version = "0.5.1", default-features = false, features = ["async_tokio"] }
+tokio = { workspace = true }
+codspeed-criterion-compat = { version = "2.6.0", default-features = false, optional = true }
+tempfile = { workspace = true }
+fs-err = { workspace = true }
+uv-resolver = { workspace = true }
+uv-cache = { workspace = true }
+uv-client = { workspace = true }
+uv-configuration = { workspace = true }
+uv-types = { workspace = true }
+uv-interpreter = { workspace = true }
 platform-tags = { workspace = true }
+pep508_rs = { workspace = true }
+anyhow = { workspace = true }
+once_cell = { workspace = true }
 
-criterion = { version = "0.5.1", default-features = false }
+[features]
+codspeed = ["codspeed-criterion-compat"]

crates/bench/benches/uv.rs

@@ -0,0 +1,189 @@
+use std::str::FromStr;
+
+use bench::criterion::black_box;
+use bench::criterion::{criterion_group, criterion_main, measurement::WallTime, Criterion};
+
+use pep508_rs::Requirement;
+use uv_cache::Cache;
+use uv_client::RegistryClientBuilder;
+use uv_resolver::Manifest;
+
+fn resolve_warm_jupyter(c: &mut Criterion<WallTime>) {
+    let runtime = &tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    let cache = &Cache::from_path(".cache").unwrap();
+    let manifest = &Manifest::simple(vec![Requirement::from_str("jupyter").unwrap()]);
+    let client = &RegistryClientBuilder::new(cache.clone()).build();
+
+    let run = || {
+        runtime
+            .block_on(resolver::resolve(
+                black_box(manifest.clone()),
+                black_box(cache.clone()),
+                black_box(client),
+            ))
+            .unwrap();
+    };
+
+    c.bench_function("resolve_warm_jupyter", |b| b.iter(run));
+}
+
+criterion_group!(uv, resolve_warm_jupyter);
+criterion_main!(uv);
+
+mod resolver {
+    use anyhow::Result;
+    use once_cell::sync::Lazy;
+    use std::path::{Path, PathBuf};
+    use std::str::FromStr;
+
+    use distribution_types::{IndexLocations, Resolution, SourceDist};
+    use pep508_rs::{MarkerEnvironment, Requirement, StringVersion};
+    use platform_tags::{Arch, Os, Platform, Tags};
+    use uv_cache::Cache;
+    use uv_client::RegistryClient;
+    use uv_configuration::{BuildKind, NoBinary, NoBuild, SetupPyStrategy};
+    use uv_interpreter::{Interpreter, PythonEnvironment};
+    use uv_resolver::{FlatIndex, InMemoryIndex, Manifest, Options, ResolutionGraph, Resolver};
+    use uv_types::{
+        BuildContext, BuildIsolation, EmptyInstalledPackages, HashStrategy, SourceBuildTrait,
+    };
+
+    static MARKERS: Lazy<MarkerEnvironment> = Lazy::new(|| {
+        MarkerEnvironment {
+            implementation_name: "cpython".to_string(),
+            implementation_version: StringVersion::from_str("3.11.5").unwrap(),
+            os_name: "posix".to_string(),
+            platform_machine: "arm64".to_string(),
+            platform_python_implementation: "CPython".to_string(),
+            platform_release: "21.6.0".to_string(),
+            platform_system: "Darwin".to_string(),
+            platform_version: "Darwin Kernel Version 21.6.0: Mon Aug 22 20:19:52 PDT 2022; root:xnu-8020.140.49~2/RELEASE_ARM64_T6000".to_string(),
+            python_full_version: StringVersion::from_str("3.11.5").unwrap(),
+            python_version: StringVersion::from_str("3.11").unwrap(),
+            sys_platform: "darwin".to_string(),
+        }
+    });
+
+    static PLATFORM: Platform = Platform::new(
+        Os::Macos {
+            major: 21,
+            minor: 6,
+        },
+        Arch::Aarch64,
+    );
+
+    static TAGS: Lazy<Tags> =
+        Lazy::new(|| Tags::from_env(&PLATFORM, (3, 11), "cpython", (3, 11), false).unwrap());
+
+    pub(crate) async fn resolve(
+        manifest: Manifest,
+        cache: Cache,
+        client: &RegistryClient,
+    ) -> Result<ResolutionGraph> {
+        let flat_index = FlatIndex::default();
+        let index = InMemoryIndex::default();
+        let interpreter = Interpreter::artificial(PLATFORM.clone(), MARKERS.clone());
+        let build_context = Context::new(cache, interpreter.clone());
+        let hashes = HashStrategy::None;
+        let installed_packages = EmptyInstalledPackages;
+
+        let resolver = Resolver::new(
+            manifest,
+            Options::default(),
+            &MARKERS,
+            &interpreter,
+            &TAGS,
+            client,
+            &flat_index,
+            &index,
+            &hashes,
+            &build_context,
+            &installed_packages,
+        )?;
+
+        Ok(resolver.resolve().await?)
+    }
+
+    struct Context {
+        cache: Cache,
+        interpreter: Interpreter,
+        index_locations: IndexLocations,
+    }
+
+    impl Context {
+        fn new(cache: Cache, interpreter: Interpreter) -> Self {
+            Self {
+                cache,
+                interpreter,
+                index_locations: IndexLocations::default(),
+            }
+        }
+    }
+
+    impl BuildContext for Context {
+        type SourceDistBuilder = DummyBuilder;
+
+        fn cache(&self) -> &Cache {
+            &self.cache
+        }
+
+        fn interpreter(&self) -> &Interpreter {
+            &self.interpreter
+        }
+
+        fn build_isolation(&self) -> BuildIsolation {
+            BuildIsolation::Isolated
+        }
+
+        fn no_build(&self) -> &NoBuild {
+            &NoBuild::None
+        }
+
+        fn no_binary(&self) -> &NoBinary {
+            &NoBinary::None
+        }
+
+        fn setup_py_strategy(&self) -> SetupPyStrategy {
+            SetupPyStrategy::default()
+        }
+
+        fn index_locations(&self) -> &IndexLocations {
+            &self.index_locations
+        }
+
+        async fn resolve<'a>(&'a self, _: &'a [Requirement]) -> Result<Resolution> {
+            panic!("benchmarks should not build source distributions")
+        }
+
+        async fn install<'a>(&'a self, _: &'a Resolution, _: &'a PythonEnvironment) -> Result<()> {
+            panic!("benchmarks should not build source distributions")
+        }
+
+        async fn setup_build<'a>(
+            &'a self,
+            _: &'a Path,
+            _: Option<&'a Path>,
+            _: &'a str,
+            _: Option<&'a SourceDist>,
+            _: BuildKind,
+        ) -> Result<Self::SourceDistBuilder> {
+            Ok(DummyBuilder)
+        }
+    }
+
+    struct DummyBuilder;
+
+    impl SourceBuildTrait for DummyBuilder {
+        async fn metadata(&mut self) -> Result<Option<PathBuf>> {
+            panic!("benchmarks should not build source distributions")
+        }
+
+        async fn wheel<'a>(&'a self, _: &'a Path) -> Result<String> {
+            panic!("benchmarks should not build source distributions")
+        }
+    }
+}

crates/bench/src/lib.rs

@@ -1,7 +1,10 @@
 pub mod criterion {
-    //! This module re-exports the criterion API unconditionally for now. It's
-    //! intended that in the future this be a way to switch the backend to
-    //! something else (like codspeed).
+    //! This module re-exports the criterion API but picks the right backend depending on whether
+    //! the benchmarks are built to run locally or with codspeed
 
+    #[cfg(not(codspeed))]
     pub use criterion::*;
+
+    #[cfg(codspeed)]
+    pub use codspeed_criterion_compat::*;
 }

crates/platform-tags/src/platform.rs

@@ -21,7 +21,7 @@ pub struct Platform {
 
 impl Platform {
     /// Create a new platform from the given operating system and architecture.
-    pub fn new(os: Os, arch: Arch) -> Self {
+    pub const fn new(os: Os, arch: Arch) -> Self {
         Self { os, arch }
     }