language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-11-02 04:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Further decrease number of cargo metadata invokes

Browse source
This commit is contained in:
Shoyu Vanilla 2025-07-07 01:24:26 +09:00
parent 8dad658bea
commit d6bdcf2f08
5 changed files with 270 additions and 243 deletions
Download patch file Download diff file Expand all files Collapse all files

395
crates/project-model/src/cargo_workspace.rs
View file

@ -300,8 +300,6 @@ pub struct CargoMetadataConfig {
pub extra_args: Vec<String>,
/// Extra env vars to set when invoking the cargo command
pub extra_env: FxHashMap<String, Option<String>>,
/// The target dir for this workspace load.
pub target_dir: Utf8PathBuf,
/// What kind of metadata are we fetching: workspace, rustc, or sysroot.
pub kind: &'static str,
/// The toolchain version, if known.
@ -317,188 +315,6 @@ struct PackageMetadata {
}
impl CargoWorkspace {
/// Fetches the metadata for the given `cargo_toml` manifest.
/// A successful result may contain another metadata error if the initial fetching failed but
/// the `--no-deps` retry succeeded.
///
/// The sysroot is used to set the `RUSTUP_TOOLCHAIN` env var when invoking cargo
/// to ensure that the rustup proxy uses the correct toolchain.
pub fn fetch_metadata(
cargo_toml: &ManifestPath,
current_dir: &AbsPath,
config: &CargoMetadataConfig,
sysroot: &Sysroot,
no_deps: bool,
locked: bool,
progress: &dyn Fn(String),
) -> anyhow::Result<(cargo_metadata::Metadata, Option<anyhow::Error>)> {
let res = Self::fetch_metadata_(
cargo_toml,
current_dir,
config,
sysroot,
no_deps,
locked,
progress,
);
if let Ok((_, Some(ref e))) = res {
tracing::warn!(
%cargo_toml,
?e,
"`cargo metadata` failed, but retry with `--no-deps` succeeded"
);
}
res
}
fn fetch_metadata_(
cargo_toml: &ManifestPath,
current_dir: &AbsPath,
config: &CargoMetadataConfig,
sysroot: &Sysroot,
no_deps: bool,
locked: bool,
progress: &dyn Fn(String),
) -> anyhow::Result<(cargo_metadata::Metadata, Option<anyhow::Error>)> {
let cargo = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
let mut meta = MetadataCommand::new();
meta.cargo_path(cargo.get_program());
cargo.get_envs().for_each(|(var, val)| _ = meta.env(var, val.unwrap_or_default()));
meta.manifest_path(cargo_toml.to_path_buf());
match &config.features {
CargoFeatures::All => {
meta.features(CargoOpt::AllFeatures);
}
CargoFeatures::Selected { features, no_default_features } => {
if *no_default_features {
meta.features(CargoOpt::NoDefaultFeatures);
}
if !features.is_empty() {
meta.features(CargoOpt::SomeFeatures(features.clone()));
}
}
}
meta.current_dir(current_dir);
let mut other_options = vec![];
// cargo metadata only supports a subset of flags of what cargo usually accepts, and usually
// the only relevant flags for metadata here are unstable ones, so we pass those along
// but nothing else
let mut extra_args = config.extra_args.iter();
while let Some(arg) = extra_args.next() {
if arg == "-Z" {
if let Some(arg) = extra_args.next() {
other_options.push("-Z".to_owned());
other_options.push(arg.to_owned());
}
}
}
if !config.targets.is_empty() {
other_options.extend(
config.targets.iter().flat_map(|it| ["--filter-platform".to_owned(), it.clone()]),
);
}
if no_deps {
other_options.push("--no-deps".to_owned());
}
let mut using_lockfile_copy = false;
// The manifest is a rust file, so this means its a script manifest
if cargo_toml.is_rust_manifest() {
other_options.push("-Zscript".to_owned());
} else if config
.toolchain_version
.as_ref()
.is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH)
{
let lockfile = <_ as AsRef<Utf8Path>>::as_ref(cargo_toml).with_extension("lock");
let target_lockfile = config
.target_dir
.join("rust-analyzer")
.join("metadata")
.join(config.kind)
.join("Cargo.lock");
match std::fs::copy(&lockfile, &target_lockfile) {
Ok(_) => {
using_lockfile_copy = true;
other_options.push("--lockfile-path".to_owned());
other_options.push(target_lockfile.to_string());
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// There exists no lockfile yet
using_lockfile_copy = true;
other_options.push("--lockfile-path".to_owned());
other_options.push(target_lockfile.to_string());
}
Err(e) => {
tracing::warn!(
"Failed to copy lock file from `{lockfile}` to `{target_lockfile}`: {e}",
);
}
}
}
if using_lockfile_copy {
other_options.push("-Zunstable-options".to_owned());
meta.env("RUSTC_BOOTSTRAP", "1");
}
// No need to lock it if we copied the lockfile, we won't modify the original after all/
// This way cargo cannot error out on us if the lockfile requires updating.
if !using_lockfile_copy && locked {
other_options.push("--locked".to_owned());
}
meta.other_options(other_options);
// FIXME: Fetching metadata is a slow process, as it might require
// calling crates.io. We should be reporting progress here, but it's
// unclear whether cargo itself supports it.
progress("cargo metadata: started".to_owned());
let res = (|| -> anyhow::Result<(_, _)> {
let mut errored = false;
let output =
spawn_with_streaming_output(meta.cargo_command(), &mut |_| (), &mut |line| {
errored = errored || line.starts_with("error") || line.starts_with("warning");
if errored {
progress("cargo metadata: ?".to_owned());
return;
}
progress(format!("cargo metadata: {line}"));
})?;
if !output.status.success() {
progress(format!("cargo metadata: failed {}", output.status));
let error = cargo_metadata::Error::CargoMetadata {
stderr: String::from_utf8(output.stderr)?,
}
.into();
if !no_deps {
// If we failed to fetch metadata with deps, try again without them.
// This makes r-a still work partially when offline.
if let Ok((metadata, _)) = Self::fetch_metadata_(
cargo_toml,
current_dir,
config,
sysroot,
true,
locked,
progress,
) {
return Ok((metadata, Some(error)));
}
}
return Err(error);
}
let stdout = from_utf8(&output.stdout)?
.lines()
.find(|line| line.starts_with('{'))
.ok_or(cargo_metadata::Error::NoJson)?;
Ok((cargo_metadata::MetadataCommand::parse(stdout)?, None))
})()
.with_context(|| format!("Failed to run `{:?}`", meta.cargo_command()));
progress("cargo metadata: finished".to_owned());
res
}
pub fn new(
mut meta: cargo_metadata::Metadata,
ws_manifest_path: ManifestPath,
@ -733,3 +549,214 @@ impl CargoWorkspace {
self.requires_rustc_private
}
}
pub(crate) struct FetchMetadata {
command: cargo_metadata::MetadataCommand,
lockfile_path: Option<Utf8PathBuf>,
kind: &'static str,
no_deps: bool,
no_deps_result: anyhow::Result<cargo_metadata::Metadata>,
other_options: Vec<String>,
}
impl FetchMetadata {
/// Builds a command to fetch metadata for the given `cargo_toml` manifest.
///
/// Performs a lightweight pre-fetch using the `--no-deps` option,
/// available via [`FetchMetadata::no_deps_metadata`], to gather basic
/// information such as the `target-dir`.
///
/// The provided sysroot is used to set the `RUSTUP_TOOLCHAIN`
/// environment variable when invoking Cargo, ensuring that the
/// rustup proxy selects the correct toolchain.
pub(crate) fn new(
cargo_toml: &ManifestPath,
current_dir: &AbsPath,
config: &CargoMetadataConfig,
sysroot: &Sysroot,
no_deps: bool,
) -> Self {
let cargo = sysroot.tool(Tool::Cargo, current_dir, &config.extra_env);
let mut command = MetadataCommand::new();
command.cargo_path(cargo.get_program());
cargo.get_envs().for_each(|(var, val)| _ = command.env(var, val.unwrap_or_default()));
command.manifest_path(cargo_toml.to_path_buf());
match &config.features {
CargoFeatures::All => {
command.features(CargoOpt::AllFeatures);
}
CargoFeatures::Selected { features, no_default_features } => {
if *no_default_features {
command.features(CargoOpt::NoDefaultFeatures);
}
if !features.is_empty() {
command.features(CargoOpt::SomeFeatures(features.clone()));
}
}
}
command.current_dir(current_dir);
let mut needs_nightly = false;
let mut other_options = vec![];
// cargo metadata only supports a subset of flags of what cargo usually accepts, and usually
// the only relevant flags for metadata here are unstable ones, so we pass those along
// but nothing else
let mut extra_args = config.extra_args.iter();
while let Some(arg) = extra_args.next() {
if arg == "-Z" {
if let Some(arg) = extra_args.next() {
needs_nightly = true;
other_options.push("-Z".to_owned());
other_options.push(arg.to_owned());
}
}
}
let mut lockfile_path = None;
if cargo_toml.is_rust_manifest() {
needs_nightly = true;
other_options.push("-Zscript".to_owned());
} else if config
.toolchain_version
.as_ref()
.is_some_and(|v| *v >= MINIMUM_TOOLCHAIN_VERSION_SUPPORTING_LOCKFILE_PATH)
{
lockfile_path = Some(<_ as AsRef<Utf8Path>>::as_ref(cargo_toml).with_extension("lock"));
}
if !config.targets.is_empty() {
other_options.extend(
config.targets.iter().flat_map(|it| ["--filter-platform".to_owned(), it.clone()]),
);
}
command.other_options(other_options.clone());
if needs_nightly {
command.env("RUSTC_BOOTSTRAP", "1");
}
// Pre-fetch basic metadata using `--no-deps`, which:
// - avoids fetching registries like crates.io,
// - skips dependency resolution and does not modify lockfiles,
// - and thus doesn't require progress reporting or copying lockfiles.
//
// Useful as a fast fallback to extract info like `target-dir`.
let cargo_command;
let no_deps_result = if no_deps {
command.no_deps();
cargo_command = command.cargo_command();
command.exec()
} else {
let mut no_deps_command = command.clone();
no_deps_command.no_deps();
cargo_command = no_deps_command.cargo_command();
no_deps_command.exec()
}
.with_context(|| format!("Failed to run `{cargo_command:?}`"));
Self { command, lockfile_path, kind: config.kind, no_deps, no_deps_result, other_options }
}
pub(crate) fn no_deps_metadata(&self) -> Option<&cargo_metadata::Metadata> {
self.no_deps_result.as_ref().ok()
}
/// Executes the metadata-fetching command.
///
/// A successful result may still contain a metadata error if the full fetch failed,
/// but the fallback `--no-deps` pre-fetch succeeded during command construction.
pub(crate) fn exec(
self,
target_dir: &Utf8Path,
locked: bool,
progress: &dyn Fn(String),
) -> anyhow::Result<(cargo_metadata::Metadata, Option<anyhow::Error>)> {
let Self { mut command, lockfile_path, kind, no_deps, no_deps_result, mut other_options } =
self;
if no_deps {
return no_deps_result.map(|m| (m, None));
}
let mut using_lockfile_copy = false;
// The manifest is a rust file, so this means its a script manifest
if let Some(lockfile) = lockfile_path {
let target_lockfile =
target_dir.join("rust-analyzer").join("metadata").join(kind).join("Cargo.lock");
match std::fs::copy(&lockfile, &target_lockfile) {
Ok(_) => {
using_lockfile_copy = true;
other_options.push("--lockfile-path".to_owned());
other_options.push(target_lockfile.to_string());
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
// There exists no lockfile yet
using_lockfile_copy = true;
other_options.push("--lockfile-path".to_owned());
other_options.push(target_lockfile.to_string());
}
Err(e) => {
tracing::warn!(
"Failed to copy lock file from `{lockfile}` to `{target_lockfile}`: {e}",
);
}
}
}
if using_lockfile_copy {
other_options.push("-Zunstable-options".to_owned());
command.env("RUSTC_BOOTSTRAP", "1");
}
// No need to lock it if we copied the lockfile, we won't modify the original after all/
// This way cargo cannot error out on us if the lockfile requires updating.
if !using_lockfile_copy && locked {
other_options.push("--locked".to_owned());
}
command.other_options(other_options);
// FIXME: Fetching metadata is a slow process, as it might require
// calling crates.io. We should be reporting progress here, but it's
// unclear whether cargo itself supports it.
progress("cargo metadata: started".to_owned());
let res = (|| -> anyhow::Result<(_, _)> {
let mut errored = false;
let output =
spawn_with_streaming_output(command.cargo_command(), &mut |_| (), &mut |line| {
errored = errored || line.starts_with("error") || line.starts_with("warning");
if errored {
progress("cargo metadata: ?".to_owned());
return;
}
progress(format!("cargo metadata: {line}"));
})?;
if !output.status.success() {
progress(format!("cargo metadata: failed {}", output.status));
let error = cargo_metadata::Error::CargoMetadata {
stderr: String::from_utf8(output.stderr)?,
}
.into();
if !no_deps {
// If we failed to fetch metadata with deps, return pre-fetched result without them.
// This makes r-a still work partially when offline.
if let Ok(metadata) = no_deps_result {
tracing::warn!(
?error,
"`cargo metadata` failed and returning succeeded result with `--no-deps`"
);
return Ok((metadata, Some(error)));
}
}
return Err(error);
}
let stdout = from_utf8(&output.stdout)?
.lines()
.find(|line| line.starts_with('{'))
.ok_or(cargo_metadata::Error::NoJson)?;
Ok((cargo_metadata::MetadataCommand::parse(stdout)?, None))
})()
.with_context(|| format!("Failed to run `{:?}`", command.cargo_command()));
progress("cargo metadata: finished".to_owned());
res
}
}

23
crates/project-model/src/sysroot.rs
View file

@ -9,14 +9,15 @@ use std::{env, fs, ops::Not, path::Path, process::Command};
use anyhow::{Result, format_err};
use itertools::Itertools;
use paths::{AbsPath, AbsPathBuf, Utf8PathBuf};
use paths::{AbsPath, AbsPathBuf, Utf8Path, Utf8PathBuf};
use rustc_hash::FxHashMap;
use stdx::format_to;
use toolchain::{Tool, probe_for_binary};
use crate::{
CargoWorkspace, ManifestPath, ProjectJson, RustSourceWorkspaceConfig,
cargo_workspace::CargoMetadataConfig, utf8_stdout,
cargo_workspace::{CargoMetadataConfig, FetchMetadata},
utf8_stdout,
};
#[derive(Debug, Clone, PartialEq, Eq)]
@ -211,6 +212,7 @@ impl Sysroot {
sysroot_source_config: &RustSourceWorkspaceConfig,
no_deps: bool,
current_dir: &AbsPath,
target_dir: &Utf8Path,
progress: &dyn Fn(String),
) -> Option<RustLibSrcWorkspace> {
assert!(matches!(self.workspace, RustLibSrcWorkspace::Empty), "workspace already loaded");
@ -224,6 +226,7 @@ impl Sysroot {
match self.load_library_via_cargo(
&library_manifest,
current_dir,
target_dir,
cargo_config,
no_deps,
progress,
@ -319,6 +322,7 @@ impl Sysroot {
&self,
library_manifest: &ManifestPath,
current_dir: &AbsPath,
target_dir: &Utf8Path,
cargo_config: &CargoMetadataConfig,
no_deps: bool,
progress: &dyn Fn(String),
@ -331,16 +335,11 @@ impl Sysroot {
Some("nightly".to_owned()),
);
let (mut res, _) = CargoWorkspace::fetch_metadata(
library_manifest,
current_dir,
&cargo_config,
self,
no_deps,
// Make sure we never attempt to write to the sysroot
true,
progress,
)?;
// Make sure we never attempt to write to the sysroot
let locked = true;
let (mut res, _) =
FetchMetadata::new(library_manifest, current_dir, &cargo_config, self, no_deps)
.exec(target_dir, locked, progress)?;
// Patch out `rustc-std-workspace-*` crates to point to the real crates.
// This is done prior to `CrateGraph` construction to prevent de-duplication logic from failing.

9
crates/project-model/src/tests.rs
View file

@ -239,8 +239,13 @@ fn smoke_test_real_sysroot_cargo() {
);
let cwd = AbsPathBuf::assert_utf8(temp_dir().join("smoke_test_real_sysroot_cargo"));
std::fs::create_dir_all(&cwd).unwrap();
let loaded_sysroot =
sysroot.load_workspace(&RustSourceWorkspaceConfig::default_cargo(), false, &cwd, &|_| ());
let loaded_sysroot = sysroot.load_workspace(
&RustSourceWorkspaceConfig::default_cargo(),
false,
&cwd,
&Utf8PathBuf::default(),
&|_| (),
);
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}

84
crates/project-model/src/workspace.rs
View file

@ -26,7 +26,7 @@ use crate::{
WorkspaceBuildScripts,
build_dependencies::BuildScriptOutput,
cargo_config_file,
cargo_workspace::{CargoMetadataConfig, DepKind, PackageData, RustLibSource},
cargo_workspace::{CargoMetadataConfig, DepKind, FetchMetadata, PackageData, RustLibSource},
env::{cargo_config_env, inject_cargo_env, inject_cargo_package_env, inject_rustc_tool_env},
project_json::{Crate, CrateArrayIdx},
sysroot::RustLibSrcWorkspace,
@ -282,10 +282,24 @@ impl ProjectWorkspace {
.ok()
.flatten();
let fetch_metadata = FetchMetadata::new(
cargo_toml,
workspace_dir,
&CargoMetadataConfig {
features: features.clone(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
toolchain_version: toolchain.clone(),
kind: "workspace",
},
&sysroot,
*no_deps,
);
let target_dir = config
.target_dir
.clone()
.or_else(|| cargo_target_dir(cargo_toml, extra_env, &sysroot))
.or_else(|| fetch_metadata.no_deps_metadata().map(|m| m.target_directory.clone()))
.unwrap_or_else(|| workspace_dir.join("target").into());
// We spawn a bunch of processes to query various information about the workspace's
@ -319,7 +333,7 @@ impl ProjectWorkspace {
};
rustc_dir.and_then(|rustc_dir| {
info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
match CargoWorkspace::fetch_metadata(
match FetchMetadata::new(
&rustc_dir,
workspace_dir,
&CargoMetadataConfig {
@ -327,15 +341,12 @@ impl ProjectWorkspace {
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
target_dir: target_dir.clone(),
toolchain_version: toolchain.clone(),
kind: "rustc-dev"
},
&sysroot,
*no_deps,
true,
progress,
) {
).exec(&target_dir, true, progress) {
Ok((meta, _error)) => {
let workspace = CargoWorkspace::new(
meta,
@ -364,35 +375,17 @@ impl ProjectWorkspace {
})
});
let cargo_metadata = s.spawn(|| {
CargoWorkspace::fetch_metadata(
cargo_toml,
workspace_dir,
&CargoMetadataConfig {
features: features.clone(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
target_dir: target_dir.clone(),
toolchain_version: toolchain.clone(),
kind: "workspace",
},
&sysroot,
*no_deps,
false,
progress,
)
});
let cargo_metadata = s.spawn(|| fetch_metadata.exec(&target_dir, false, progress));
let loaded_sysroot = s.spawn(|| {
sysroot.load_workspace(
&RustSourceWorkspaceConfig::CargoMetadata(sysroot_metadata_config(
config,
&targets,
toolchain.clone(),
target_dir.clone(),
)),
config.no_deps,
workspace_dir,
&target_dir,
progress,
)
});
@ -500,6 +493,7 @@ impl ProjectWorkspace {
&RustSourceWorkspaceConfig::Json(*sysroot_project),
config.no_deps,
project_root,
&target_dir,
progress,
)
} else {
@ -508,10 +502,10 @@ impl ProjectWorkspace {
config,
&targets,
toolchain.clone(),
target_dir,
)),
config.no_deps,
project_root,
&target_dir,
progress,
)
}
@ -570,17 +564,17 @@ impl ProjectWorkspace {
config,
&targets,
toolchain.clone(),
target_dir.clone(),
)),
config.no_deps,
dir,
&target_dir,
&|_| (),
);
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let cargo_script = CargoWorkspace::fetch_metadata(
let fetch_metadata = FetchMetadata::new(
detached_file,
dir,
&CargoMetadataConfig {
@ -588,24 +582,26 @@ impl ProjectWorkspace {
targets,
extra_args: config.extra_args.clone(),
extra_env: config.extra_env.clone(),
target_dir,
toolchain_version: toolchain.clone(),
kind: "detached-file",
},
&sysroot,
config.no_deps,
false,
&|_| (),
)
.ok()
.map(|(ws, error)| {
let cargo_config_extra_env = cargo_config_env(detached_file, &config_file);
(
CargoWorkspace::new(ws, detached_file.clone(), cargo_config_extra_env, false),
WorkspaceBuildScripts::default(),
error.map(Arc::new),
)
});
);
let target_dir = config
.target_dir
.clone()
.or_else(|| fetch_metadata.no_deps_metadata().map(|m| m.target_directory.clone()))
.unwrap_or_else(|| dir.join("target").into());
let cargo_script =
fetch_metadata.exec(&target_dir, false, &|_| ()).ok().map(|(ws, error)| {
let cargo_config_extra_env = cargo_config_env(detached_file, &config_file);
(
CargoWorkspace::new(ws, detached_file.clone(), cargo_config_extra_env, false),
WorkspaceBuildScripts::default(),
error.map(Arc::new),
)
});
Ok(ProjectWorkspace {
kind: ProjectWorkspaceKind::DetachedFile {
@ -1887,14 +1883,12 @@ fn sysroot_metadata_config(
config: &CargoConfig,
targets: &[String],
toolchain_version: Option<Version>,
target_dir: Utf8PathBuf,
) -> CargoMetadataConfig {
CargoMetadataConfig {
features: Default::default(),
targets: targets.to_vec(),
extra_args: Default::default(),
extra_env: config.extra_env.clone(),
target_dir,
toolchain_version,
kind: "sysroot",
}