From d6bdcf2f08b21e59a9bdb690d47b2288dac89176 Mon Sep 17 00:00:00 2001 From: Shoyu Vanilla Date: Mon, 7 Jul 2025 01:24:26 +0900 Subject: [PATCH] Further decrease number of `cargo metadata` invokes --- crates/project-model/src/cargo_workspace.rs | 395 +++++++++++--------- crates/project-model/src/sysroot.rs | 23 +- crates/project-model/src/tests.rs | 9 +- crates/project-model/src/workspace.rs | 84 ++--- crates/rust-analyzer/src/cli/rustc_tests.rs | 2 + 5 files changed, 270 insertions(+), 243 deletions(-) diff --git a/crates/project-model/src/cargo_workspace.rs b/crates/project-model/src/cargo_workspace.rs index 4bacc90417..daadcd9d79 100644 --- a/crates/project-model/src/cargo_workspace.rs +++ b/crates/project-model/src/cargo_workspace.rs @@ -300,8 +300,6 @@ pub struct CargoMetadataConfig { pub extra_args: Vec, /// Extra env vars to set when invoking the cargo command pub extra_env: FxHashMap>, - /// The target dir for this workspace load. - pub target_dir: Utf8PathBuf, /// What kind of metadata are we fetching: workspace, rustc, or sysroot. pub kind: &'static str, /// The toolchain version, if known. @@ -317,188 +315,6 @@ struct PackageMetadata { } impl CargoWorkspace { - /// Fetches the metadata for the given `cargo_toml` manifest. - /// A successful result may contain another metadata error if the initial fetching failed but - /// the `--no-deps` retry succeeded. - /// - /// The sysroot is used to set the `RUSTUP_TOOLCHAIN` env var when invoking cargo - /// to ensure that the rustup proxy uses the correct toolchain. - pub fn fetch_metadata( - cargo_toml: &ManifestPath, - current_dir: &AbsPath, - config: &CargoMetadataConfig, - sysroot: &Sysroot, - no_deps: bool, - locked: bool, - progress: &dyn Fn(String), - ) -> anyhow::Result<(cargo_metadata::Metadata, Option)> { - let res = Self::fetch_metadata_( - cargo_toml, - current_dir, - config, - sysroot, - no_deps, - locked, - progress, - ); - if let Ok((_, Some(ref e))) = res { - tracing::warn!( - %cargo_toml, - ?e, - "`cargo metadata` failed, but retry with `--no-deps` succeeded" - ); - } - res - } - - fn fetch_metadata_( - cargo_toml: &ManifestPath, - current_dir: &AbsPath, - config: &CargoMetadataConfig, - sysroot: &Sysroot, - no_deps: bool, - locked: bool, - progress: &dyn Fn(String), - ) -> anyhow::Result<(cargo_metadata::Metadata, Option)> { - let cargo = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env); - let mut meta = MetadataCommand::new(); - meta.cargo_path(cargo.get_program()); - cargo.get_envs().for_each(|(var, val)| _ = meta.env(var, val.unwrap_or_default())); - meta.manifest_path(cargo_toml.to_path_buf()); - match &config.features { - CargoFeatures::All => { - meta.features(CargoOpt::AllFeatures); - } - CargoFeatures::Selected { features, no_default_features } => { - if *no_default_features { - meta.features(CargoOpt::NoDefaultFeatures); - } - if !features.is_empty() { - meta.features(CargoOpt::SomeFeatures(features.clone())); - } - } - } - meta.current_dir(current_dir); - - let mut other_options = vec![]; - // cargo metadata only supports a subset of flags of what cargo usually accepts, and usually - // the only relevant flags for metadata here are unstable ones, so we pass those along - // but nothing else - let mut extra_args = config.extra_args.iter(); - while let Some(arg) = extra_args.next() { - if arg == "-Z" { - if let Some(arg) = extra_args.next() { - other_options.push("-Z".to_owned()); - other_options.push(arg.to_owned()); - } - } - } - - if !config.targets.is_empty() { - other_options.extend( - config.targets.iter().flat_map(|it| ["--filter-platform".to_owned(), it.clone()]), - ); - } - if no_deps { - other_options.push("--no-deps".to_owned()); - } - - let mut using_lockfile_copy = false; - // The manifest is a rust file, so this means its a script manifest - if cargo_toml.is_rust_manifest() { - other_options.push("-Zscript".to_owned()); - } else if config - .toolchain_version - .as_ref() - .is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH) - { - let lockfile = <_ as AsRef>::as_ref(cargo_toml).with_extension("lock"); - let target_lockfile = config - .target_dir - .join("rust-analyzer") - .join("metadata") - .join(config.kind) - .join("Cargo.lock"); - match std::fs::copy(&lockfile, &target_lockfile) { - Ok(_) => { - using_lockfile_copy = true; - other_options.push("--lockfile-path".to_owned()); - other_options.push(target_lockfile.to_string()); - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - // There exists no lockfile yet - using_lockfile_copy = true; - other_options.push("--lockfile-path".to_owned()); - other_options.push(target_lockfile.to_string()); - } - Err(e) => { - tracing::warn!( - "Failed to copy lock file from `{lockfile}` to `{target_lockfile}`: {e}", - ); - } - } - } - if using_lockfile_copy { - other_options.push("-Zunstable-options".to_owned()); - meta.env("RUSTC_BOOTSTRAP", "1"); - } - // No need to lock it if we copied the lockfile, we won't modify the original after all/ - // This way cargo cannot error out on us if the lockfile requires updating. - if !using_lockfile_copy && locked { - other_options.push("--locked".to_owned()); - } - meta.other_options(other_options); - - // FIXME: Fetching metadata is a slow process, as it might require - // calling crates.io. We should be reporting progress here, but it's - // unclear whether cargo itself supports it. - progress("cargo metadata: started".to_owned()); - - let res = (|| -> anyhow::Result<(_, _)> { - let mut errored = false; - let output = - spawn_with_streaming_output(meta.cargo_command(), &mut |_| (), &mut |line| { - errored = errored || line.starts_with("error") || line.starts_with("warning"); - if errored { - progress("cargo metadata: ?".to_owned()); - return; - } - progress(format!("cargo metadata: {line}")); - })?; - if !output.status.success() { - progress(format!("cargo metadata: failed {}", output.status)); - let error = cargo_metadata::Error::CargoMetadata { - stderr: String::from_utf8(output.stderr)?, - } - .into(); - if !no_deps { - // If we failed to fetch metadata with deps, try again without them. - // This makes r-a still work partially when offline. - if let Ok((metadata, _)) = Self::fetch_metadata_( - cargo_toml, - current_dir, - config, - sysroot, - true, - locked, - progress, - ) { - return Ok((metadata, Some(error))); - } - } - return Err(error); - } - let stdout = from_utf8(&output.stdout)? - .lines() - .find(|line| line.starts_with('{')) - .ok_or(cargo_metadata::Error::NoJson)?; - Ok((cargo_metadata::MetadataCommand::parse(stdout)?, None)) - })() - .with_context(|| format!("Failed to run `{:?}`", meta.cargo_command())); - progress("cargo metadata: finished".to_owned()); - res - } - pub fn new( mut meta: cargo_metadata::Metadata, ws_manifest_path: ManifestPath, @@ -733,3 +549,214 @@ impl CargoWorkspace { self.requires_rustc_private } } + +pub(crate) struct FetchMetadata { + command: cargo_metadata::MetadataCommand, + lockfile_path: Option, + kind: &'static str, + no_deps: bool, + no_deps_result: anyhow::Result, + other_options: Vec, +} + +impl FetchMetadata { + /// Builds a command to fetch metadata for the given `cargo_toml` manifest. + /// + /// Performs a lightweight pre-fetch using the `--no-deps` option, + /// available via [`FetchMetadata::no_deps_metadata`], to gather basic + /// information such as the `target-dir`. + /// + /// The provided sysroot is used to set the `RUSTUP_TOOLCHAIN` + /// environment variable when invoking Cargo, ensuring that the + /// rustup proxy selects the correct toolchain. + pub(crate) fn new( + cargo_toml: &ManifestPath, + current_dir: &AbsPath, + config: &CargoMetadataConfig, + sysroot: &Sysroot, + no_deps: bool, + ) -> Self { + let cargo = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env); + let mut command = MetadataCommand::new(); + command.cargo_path(cargo.get_program()); + cargo.get_envs().for_each(|(var, val)| _ = command.env(var, val.unwrap_or_default())); + command.manifest_path(cargo_toml.to_path_buf()); + match &config.features { + CargoFeatures::All => { + command.features(CargoOpt::AllFeatures); + } + CargoFeatures::Selected { features, no_default_features } => { + if *no_default_features { + command.features(CargoOpt::NoDefaultFeatures); + } + if !features.is_empty() { + command.features(CargoOpt::SomeFeatures(features.clone())); + } + } + } + command.current_dir(current_dir); + + let mut needs_nightly = false; + let mut other_options = vec![]; + // cargo metadata only supports a subset of flags of what cargo usually accepts, and usually + // the only relevant flags for metadata here are unstable ones, so we pass those along + // but nothing else + let mut extra_args = config.extra_args.iter(); + while let Some(arg) = extra_args.next() { + if arg == "-Z" { + if let Some(arg) = extra_args.next() { + needs_nightly = true; + other_options.push("-Z".to_owned()); + other_options.push(arg.to_owned()); + } + } + } + + let mut lockfile_path = None; + if cargo_toml.is_rust_manifest() { + needs_nightly = true; + other_options.push("-Zscript".to_owned()); + } else if config + .toolchain_version + .as_ref() + .is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH) + { + lockfile_path = Some(<_ as AsRef>::as_ref(cargo_toml).with_extension("lock")); + } + + if !config.targets.is_empty() { + other_options.extend( + config.targets.iter().flat_map(|it| ["--filter-platform".to_owned(), it.clone()]), + ); + } + + command.other_options(other_options.clone()); + + if needs_nightly { + command.env("RUSTC_BOOTSTRAP", "1"); + } + + // Pre-fetch basic metadata using `--no-deps`, which: + // - avoids fetching registries like crates.io, + // - skips dependency resolution and does not modify lockfiles, + // - and thus doesn't require progress reporting or copying lockfiles. + // + // Useful as a fast fallback to extract info like `target-dir`. + let cargo_command; + let no_deps_result = if no_deps { + command.no_deps(); + cargo_command = command.cargo_command(); + command.exec() + } else { + let mut no_deps_command = command.clone(); + no_deps_command.no_deps(); + cargo_command = no_deps_command.cargo_command(); + no_deps_command.exec() + } + .with_context(|| format!("Failed to run `{cargo_command:?}`")); + + Self { command, lockfile_path, kind: config.kind, no_deps, no_deps_result, other_options } + } + + pub(crate) fn no_deps_metadata(&self) -> Option<&cargo_metadata::Metadata> { + self.no_deps_result.as_ref().ok() + } + + /// Executes the metadata-fetching command. + /// + /// A successful result may still contain a metadata error if the full fetch failed, + /// but the fallback `--no-deps` pre-fetch succeeded during command construction. + pub(crate) fn exec( + self, + target_dir: &Utf8Path, + locked: bool, + progress: &dyn Fn(String), + ) -> anyhow::Result<(cargo_metadata::Metadata, Option)> { + let Self { mut command, lockfile_path, kind, no_deps, no_deps_result, mut other_options } = + self; + + if no_deps { + return no_deps_result.map(|m| (m, None)); + } + + let mut using_lockfile_copy = false; + // The manifest is a rust file, so this means its a script manifest + if let Some(lockfile) = lockfile_path { + let target_lockfile = + target_dir.join("rust-analyzer").join("metadata").join(kind).join("Cargo.lock"); + match std::fs::copy(&lockfile, &target_lockfile) { + Ok(_) => { + using_lockfile_copy = true; + other_options.push("--lockfile-path".to_owned()); + other_options.push(target_lockfile.to_string()); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // There exists no lockfile yet + using_lockfile_copy = true; + other_options.push("--lockfile-path".to_owned()); + other_options.push(target_lockfile.to_string()); + } + Err(e) => { + tracing::warn!( + "Failed to copy lock file from `{lockfile}` to `{target_lockfile}`: {e}", + ); + } + } + } + if using_lockfile_copy { + other_options.push("-Zunstable-options".to_owned()); + command.env("RUSTC_BOOTSTRAP", "1"); + } + // No need to lock it if we copied the lockfile, we won't modify the original after all/ + // This way cargo cannot error out on us if the lockfile requires updating. + if !using_lockfile_copy && locked { + other_options.push("--locked".to_owned()); + } + command.other_options(other_options); + + // FIXME: Fetching metadata is a slow process, as it might require + // calling crates.io. We should be reporting progress here, but it's + // unclear whether cargo itself supports it. + progress("cargo metadata: started".to_owned()); + + let res = (|| -> anyhow::Result<(_, _)> { + let mut errored = false; + let output = + spawn_with_streaming_output(command.cargo_command(), &mut |_| (), &mut |line| { + errored = errored || line.starts_with("error") || line.starts_with("warning"); + if errored { + progress("cargo metadata: ?".to_owned()); + return; + } + progress(format!("cargo metadata: {line}")); + })?; + if !output.status.success() { + progress(format!("cargo metadata: failed {}", output.status)); + let error = cargo_metadata::Error::CargoMetadata { + stderr: String::from_utf8(output.stderr)?, + } + .into(); + if !no_deps { + // If we failed to fetch metadata with deps, return pre-fetched result without them. + // This makes r-a still work partially when offline. + if let Ok(metadata) = no_deps_result { + tracing::warn!( + ?error, + "`cargo metadata` failed and returning succeeded result with `--no-deps`" + ); + return Ok((metadata, Some(error))); + } + } + return Err(error); + } + let stdout = from_utf8(&output.stdout)? + .lines() + .find(|line| line.starts_with('{')) + .ok_or(cargo_metadata::Error::NoJson)?; + Ok((cargo_metadata::MetadataCommand::parse(stdout)?, None)) + })() + .with_context(|| format!("Failed to run `{:?}`", command.cargo_command())); + progress("cargo metadata: finished".to_owned()); + res + } +} diff --git a/crates/project-model/src/sysroot.rs b/crates/project-model/src/sysroot.rs index 9f19260d30..9781c46737 100644 --- a/crates/project-model/src/sysroot.rs +++ b/crates/project-model/src/sysroot.rs @@ -9,14 +9,15 @@ use std::{env, fs, ops::Not, path::Path, process::Command}; use anyhow::{Result, format_err}; use itertools::Itertools; -use paths::{AbsPath, AbsPathBuf, Utf8PathBuf}; +use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf}; use rustc_hash::FxHashMap; use stdx::format_to; use toolchain::{Tool, probe_for_binary}; use crate::{ CargoWorkspace, ManifestPath, ProjectJson, RustSourceWorkspaceConfig, - cargo_workspace::CargoMetadataConfig, utf8_stdout, + cargo_workspace::{CargoMetadataConfig, FetchMetadata}, + utf8_stdout, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -211,6 +212,7 @@ impl Sysroot { sysroot_source_config: &RustSourceWorkspaceConfig, no_deps: bool, current_dir: &AbsPath, + target_dir: &Utf8Path, progress: &dyn Fn(String), ) -> Option { assert!(matches!(self.workspace, RustLibSrcWorkspace::Empty), "workspace already loaded"); @@ -224,6 +226,7 @@ impl Sysroot { match self.load_library_via_cargo( &library_manifest, current_dir, + target_dir, cargo_config, no_deps, progress, @@ -319,6 +322,7 @@ impl Sysroot { &self, library_manifest: &ManifestPath, current_dir: &AbsPath, + target_dir: &Utf8Path, cargo_config: &CargoMetadataConfig, no_deps: bool, progress: &dyn Fn(String), @@ -331,16 +335,11 @@ impl Sysroot { Some("nightly".to_owned()), ); - let (mut res, _) = CargoWorkspace::fetch_metadata( - library_manifest, - current_dir, - &cargo_config, - self, - no_deps, - // Make sure we never attempt to write to the sysroot - true, - progress, - )?; + // Make sure we never attempt to write to the sysroot + let locked = true; + let (mut res, _) = + FetchMetadata::new(library_manifest, current_dir, &cargo_config, self, no_deps) + .exec(target_dir, locked, progress)?; // Patch out `rustc-std-workspace-*` crates to point to the real crates. // This is done prior to `CrateGraph` construction to prevent de-duplication logic from failing. diff --git a/crates/project-model/src/tests.rs b/crates/project-model/src/tests.rs index f229e9a650..ed72520f40 100644 --- a/crates/project-model/src/tests.rs +++ b/crates/project-model/src/tests.rs @@ -239,8 +239,13 @@ fn smoke_test_real_sysroot_cargo() { ); let cwd = AbsPathBuf::assert_utf8(temp_dir().join("smoke_test_real_sysroot_cargo")); std::fs::create_dir_all(&cwd).unwrap(); - let loaded_sysroot = - sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo(), false, &cwd, &|_| ()); + let loaded_sysroot = sysroot.load_workspace( + &RustSourceWorkspaceConfig::default_cargo(), + false, + &cwd, + &Utf8PathBuf::default(), + &|_| (), + ); if let Some(loaded_sysroot) = loaded_sysroot { sysroot.set_workspace(loaded_sysroot); } diff --git a/crates/project-model/src/workspace.rs b/crates/project-model/src/workspace.rs index 801c2abdfc..677f29e3c6 100644 --- a/crates/project-model/src/workspace.rs +++ b/crates/project-model/src/workspace.rs @@ -26,7 +26,7 @@ use crate::{ WorkspaceBuildScripts, build_dependencies::BuildScriptOutput, cargo_config_file, - cargo_workspace::{CargoMetadataConfig, DepKind, PackageData, RustLibSource}, + cargo_workspace::{CargoMetadataConfig, DepKind, FetchMetadata, PackageData, RustLibSource}, env::{cargo_config_env, inject_cargo_env, inject_cargo_package_env, inject_rustc_tool_env}, project_json::{Crate, CrateArrayIdx}, sysroot::RustLibSrcWorkspace, @@ -282,10 +282,24 @@ impl ProjectWorkspace { .ok() .flatten(); + let fetch_metadata = FetchMetadata::new( + cargo_toml, + workspace_dir, + &CargoMetadataConfig { + features: features.clone(), + targets: targets.clone(), + extra_args: extra_args.clone(), + extra_env: extra_env.clone(), + toolchain_version: toolchain.clone(), + kind: "workspace", + }, + &sysroot, + *no_deps, + ); let target_dir = config .target_dir .clone() - .or_else(|| cargo_target_dir(cargo_toml, extra_env, &sysroot)) + .or_else(|| fetch_metadata.no_deps_metadata().map(|m| m.target_directory.clone())) .unwrap_or_else(|| workspace_dir.join("target").into()); // We spawn a bunch of processes to query various information about the workspace's @@ -319,7 +333,7 @@ impl ProjectWorkspace { }; rustc_dir.and_then(|rustc_dir| { info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source"); - match CargoWorkspace::fetch_metadata( + match FetchMetadata::new( &rustc_dir, workspace_dir, &CargoMetadataConfig { @@ -327,15 +341,12 @@ impl ProjectWorkspace { targets: targets.clone(), extra_args: extra_args.clone(), extra_env: extra_env.clone(), - target_dir: target_dir.clone(), toolchain_version: toolchain.clone(), kind: "rustc-dev" }, &sysroot, *no_deps, - true, - progress, - ) { + ).exec(&target_dir, true, progress) { Ok((meta, _error)) => { let workspace = CargoWorkspace::new( meta, @@ -364,35 +375,17 @@ impl ProjectWorkspace { }) }); - let cargo_metadata = s.spawn(|| { - CargoWorkspace::fetch_metadata( - cargo_toml, - workspace_dir, - &CargoMetadataConfig { - features: features.clone(), - targets: targets.clone(), - extra_args: extra_args.clone(), - extra_env: extra_env.clone(), - target_dir: target_dir.clone(), - toolchain_version: toolchain.clone(), - kind: "workspace", - }, - &sysroot, - *no_deps, - false, - progress, - ) - }); + let cargo_metadata = s.spawn(|| fetch_metadata.exec(&target_dir, false, progress)); let loaded_sysroot = s.spawn(|| { sysroot.load_workspace( &RustSourceWorkspaceConfig::CargoMetadata(sysroot_metadata_config( config, &targets, toolchain.clone(), - target_dir.clone(), )), config.no_deps, workspace_dir, + &target_dir, progress, ) }); @@ -500,6 +493,7 @@ impl ProjectWorkspace { &RustSourceWorkspaceConfig::Json(*sysroot_project), config.no_deps, project_root, + &target_dir, progress, ) } else { @@ -508,10 +502,10 @@ impl ProjectWorkspace { config, &targets, toolchain.clone(), - target_dir, )), config.no_deps, project_root, + &target_dir, progress, ) } @@ -570,17 +564,17 @@ impl ProjectWorkspace { config, &targets, toolchain.clone(), - target_dir.clone(), )), config.no_deps, dir, + &target_dir, &|_| (), ); if let Some(loaded_sysroot) = loaded_sysroot { sysroot.set_workspace(loaded_sysroot); } - let cargo_script = CargoWorkspace::fetch_metadata( + let fetch_metadata = FetchMetadata::new( detached_file, dir, &CargoMetadataConfig { @@ -588,24 +582,26 @@ impl ProjectWorkspace { targets, extra_args: config.extra_args.clone(), extra_env: config.extra_env.clone(), - target_dir, toolchain_version: toolchain.clone(), kind: "detached-file", }, &sysroot, config.no_deps, - false, - &|_| (), - ) - .ok() - .map(|(ws, error)| { - let cargo_config_extra_env = cargo_config_env(detached_file, &config_file); - ( - CargoWorkspace::new(ws, detached_file.clone(), cargo_config_extra_env, false), - WorkspaceBuildScripts::default(), - error.map(Arc::new), - ) - }); + ); + let target_dir = config + .target_dir + .clone() + .or_else(|| fetch_metadata.no_deps_metadata().map(|m| m.target_directory.clone())) + .unwrap_or_else(|| dir.join("target").into()); + let cargo_script = + fetch_metadata.exec(&target_dir, false, &|_| ()).ok().map(|(ws, error)| { + let cargo_config_extra_env = cargo_config_env(detached_file, &config_file); + ( + CargoWorkspace::new(ws, detached_file.clone(), cargo_config_extra_env, false), + WorkspaceBuildScripts::default(), + error.map(Arc::new), + ) + }); Ok(ProjectWorkspace { kind: ProjectWorkspaceKind::DetachedFile { @@ -1887,14 +1883,12 @@ fn sysroot_metadata_config( config: &CargoConfig, targets: &[String], toolchain_version: Option, - target_dir: Utf8PathBuf, ) -> CargoMetadataConfig { CargoMetadataConfig { features: Default::default(), targets: targets.to_vec(), extra_args: Default::default(), extra_env: config.extra_env.clone(), - target_dir, toolchain_version, kind: "sysroot", } diff --git a/crates/rust-analyzer/src/cli/rustc_tests.rs b/crates/rust-analyzer/src/cli/rustc_tests.rs index f97bf83244..30ac93fb6f 100644 --- a/crates/rust-analyzer/src/cli/rustc_tests.rs +++ b/crates/rust-analyzer/src/cli/rustc_tests.rs @@ -9,6 +9,7 @@ use hir::{ChangeWithProcMacros, Crate}; use ide::{AnalysisHost, DiagnosticCode, DiagnosticsConfig}; use ide_db::base_db; use itertools::Either; +use paths::Utf8PathBuf; use profile::StopWatch; use project_model::toolchain_info::{QueryConfig, target_data_layout}; use project_model::{ @@ -79,6 +80,7 @@ impl Tester { &RustSourceWorkspaceConfig::default_cargo(), false, &path, + &Utf8PathBuf::default(), &|_| (), ); if let Some(loaded_sysroot) = loaded_sysroot {