language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-13 13:57:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Spawn toolchain querying processes in parallel

Browse source
This commit is contained in:
Lukas Wirth 2025-02-12 08:30:30 +01:00
parent 8aa4ae5e69
commit 8a8ff99273
6 changed files with 229 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

23
crates/project-model/src/sysroot.rs
View file

@ -34,14 +34,14 @@ pub struct Sysroot {
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) enum SysrootWorkspace {
pub enum SysrootWorkspace {
Workspace(CargoWorkspace),
Stitched(Stitched),
Empty,
}
#[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) struct Stitched {
pub struct Stitched {
crates: Arena<SysrootCrateData>,
}
@ -227,18 +227,21 @@ impl Sysroot {
}
}
pub fn load_workspace(&mut self, sysroot_source_config: &SysrootSourceWorkspaceConfig) {
pub fn load_workspace(
&self,
sysroot_source_config: &SysrootSourceWorkspaceConfig,
) -> Option<SysrootWorkspace> {
assert!(matches!(self.workspace, SysrootWorkspace::Empty), "workspace already loaded");
let Self { root: _, src_root: Some(src_root), workspace, error: _ } = self else { return };
let Self { root: _, src_root: Some(src_root), workspace: _, error: _ } = self else {
return None;
};
if let SysrootSourceWorkspaceConfig::CargoMetadata(cargo_config) = sysroot_source_config {
let library_manifest = ManifestPath::try_from(src_root.join("Cargo.toml")).unwrap();
if fs::metadata(&library_manifest).is_ok() {
if let Some(loaded) =
Self::load_library_via_cargo(library_manifest, src_root, cargo_config)
{
*workspace = loaded;
self.load_core_check();
return;
return Some(loaded);
}
}
}
@ -286,11 +289,11 @@ impl Sysroot {
}
}
}
*workspace = SysrootWorkspace::Stitched(stitched);
self.load_core_check();
Some(SysrootWorkspace::Stitched(stitched))
}
fn load_core_check(&mut self) {
pub fn set_workspace(&mut self, workspace: SysrootWorkspace) {
self.workspace = workspace;
if self.error.is_none() {
if let Some(src_root) = &self.src_root {
let has_core = match &self.workspace {

10
crates/project-model/src/tests.rs
View file

@ -125,7 +125,10 @@ fn get_fake_sysroot() -> Sysroot {
let sysroot_dir = AbsPathBuf::assert(sysroot_path);
let sysroot_src_dir = sysroot_dir.clone();
let mut sysroot = Sysroot::new(Some(sysroot_dir), Some(sysroot_src_dir));
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
sysroot
}
@ -271,7 +274,10 @@ fn smoke_test_real_sysroot_cargo() {
AbsPath::assert(Utf8Path::new(env!("CARGO_MANIFEST_DIR"))),
&Default::default(),
);
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
assert!(matches!(sysroot.workspace(), SysrootWorkspace::Workspace(_)));
let project_workspace = ProjectWorkspace {
kind: ProjectWorkspaceKind::Cargo {

296
crates/project-model/src/workspace.rs
View file

@ -2,7 +2,7 @@
//! metadata` or `rust-project.json`) into representation stored in the salsa
//! database -- `CrateGraph`.
use std::{collections::VecDeque, fmt, fs, iter, ops::Deref, sync};
use std::{collections::VecDeque, fmt, fs, iter, ops::Deref, sync, thread};
use anyhow::Context;
use base_db::{
@ -186,7 +186,7 @@ impl ProjectWorkspace {
let project_location = project_json.parent().to_path_buf();
let project_json: ProjectJson =
ProjectJson::new(Some(project_json.clone()), &project_location, data);
ProjectWorkspace::load_inline(project_json, config)
ProjectWorkspace::load_inline(project_json, config, progress)
}
ProjectManifest::CargoScript(rust_file) => {
ProjectWorkspace::load_detached_file(rust_file, config)?
@ -204,14 +204,28 @@ impl ProjectWorkspace {
config: &CargoConfig,
progress: &dyn Fn(String),
) -> Result<ProjectWorkspace, anyhow::Error> {
let mut sysroot = match (&config.sysroot, &config.sysroot_src) {
progress("Discovering sysroot".to_owned());
let CargoConfig {
features,
rustc_source,
extra_args,
extra_env,
set_test,
cfg_overrides,
extra_includes,
sysroot,
sysroot_src,
target,
..
} = config;
let mut sysroot = match (sysroot, sysroot_src) {
(Some(RustLibSource::Discover), None) => {
Sysroot::discover(cargo_toml.parent(), &config.extra_env)
Sysroot::discover(cargo_toml.parent(), extra_env)
}
(Some(RustLibSource::Discover), Some(sysroot_src)) => {
Sysroot::discover_with_src_override(
cargo_toml.parent(),
&config.extra_env,
extra_env,
sysroot_src.clone(),
)
}
@ -224,100 +238,147 @@ impl ProjectWorkspace {
(None, _) => Sysroot::empty(),
};
let rustc_dir = match &config.rustc_source {
Some(RustLibSource::Path(path)) => ManifestPath::try_from(path.clone())
.map_err(|p| Some(format!("rustc source path is not absolute: {p}"))),
Some(RustLibSource::Discover) => sysroot
.discover_rustc_src()
.ok_or_else(|| Some("Failed to discover rustc source for sysroot.".to_owned())),
None => Err(None),
};
tracing::info!(workspace = %cargo_toml, src_root = ?sysroot.src_root(), root = ?sysroot.root(), "Using sysroot");
progress("Querying project metadata".to_owned());
let toolchain_config = QueryConfig::Cargo(&sysroot, cargo_toml);
let targets =
target_tuple::get(toolchain_config, config.target.as_deref(), &config.extra_env)
.unwrap_or_default();
let toolchain = version::get(toolchain_config, &config.extra_env)
.inspect_err(|e| {
tracing::error!(%e,
"failed fetching toolchain version for {cargo_toml:?} workspace"
)
})
.ok()
.flatten();
let rustc_cfg =
rustc_cfg::get(toolchain_config, targets.first().map(Deref::deref), &config.extra_env);
let cfg_overrides = config.cfg_overrides.clone();
let data_layout = target_data_layout::get(
toolchain_config,
targets.first().map(Deref::deref),
&config.extra_env,
);
if let Err(e) = &data_layout {
tracing::error!(%e, "failed fetching data layout for {cargo_toml:?} workspace");
}
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(&config.extra_env, &targets),
));
target_tuple::get(toolchain_config, target.as_deref(), extra_env).unwrap_or_default();
let rustc = rustc_dir.and_then(|rustc_dir| {
info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
match CargoWorkspace::fetch_metadata(
&rustc_dir,
cargo_toml.parent(),
&CargoMetadataConfig {
features: crate::CargoFeatures::default(),
targets: targets.clone(),
extra_args: config.extra_args.clone(),
extra_env: config.extra_env.clone(),
},
&sysroot,
false,
progress,
) {
Ok((meta, _error)) => {
let workspace = CargoWorkspace::new(meta, cargo_toml.clone(), Env::default());
let build_scripts = WorkspaceBuildScripts::rustc_crates(
&workspace,
cargo_toml.parent(),
&config.extra_env,
// We spawn a bunch of processes to query various information about the workspace's
// toolchain and sysroot
// We can speed up loading a bit by spawning all of these processes in parallel (especially
// on systems were process spawning is delayed)
let join = thread::scope(|s| {
let workspace_dir = cargo_toml.parent();
let toolchain = s.spawn(|| {
version::get(toolchain_config, extra_env)
.inspect_err(|e| {
tracing::error!(%e,
"failed fetching toolchain version for {cargo_toml:?} workspace"
)
})
.ok()
.flatten()
});
let rustc_cfg = s.spawn(|| {
rustc_cfg::get(toolchain_config, targets.first().map(Deref::deref), extra_env)
});
let data_layout = s.spawn(|| {
target_data_layout::get(
toolchain_config,
targets.first().map(Deref::deref),
extra_env,
).inspect_err(|e| {
tracing::error!(%e, "failed fetching data layout for {cargo_toml:?} workspace")
})
});
let rustc_dir = s.spawn(|| {
let rustc_dir = match rustc_source {
Some(RustLibSource::Path(path)) => ManifestPath::try_from(path.clone())
.map_err(|p| Some(format!("rustc source path is not absolute: {p}"))),
Some(RustLibSource::Discover) => {
sysroot.discover_rustc_src().ok_or_else(|| {
Some("Failed to discover rustc source for sysroot.".to_owned())
})
}
None => Err(None),
};
rustc_dir.and_then(|rustc_dir| {
info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
match CargoWorkspace::fetch_metadata(
&rustc_dir,
workspace_dir,
&CargoMetadataConfig {
features: crate::CargoFeatures::default(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
},
&sysroot,
);
Ok(Box::new((workspace, build_scripts)))
}
Err(e) => {
tracing::error!(
%e,
"Failed to read Cargo metadata from rustc source at {rustc_dir}",
);
Err(Some(format!(
"Failed to read Cargo metadata from rustc source at {rustc_dir}: {e}"
)))
}
}
false,
&|_| (),
) {
Ok((meta, _error)) => {
let workspace =
CargoWorkspace::new(meta, cargo_toml.clone(), Env::default());
let build_scripts = WorkspaceBuildScripts::rustc_crates(
&workspace,
workspace_dir,
extra_env,
&sysroot,
);
Ok(Box::new((workspace, build_scripts)))
}
Err(e) => {
tracing::error!(
%e,
"Failed to read Cargo metadata from rustc source at {rustc_dir}",
);
Err(Some(format!(
"Failed to read Cargo metadata from rustc source at {rustc_dir}: {e}"
)))
}
}
})
});
let cargo_metadata = s.spawn(|| {
CargoWorkspace::fetch_metadata(
cargo_toml,
workspace_dir,
&CargoMetadataConfig {
features: features.clone(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
},
&sysroot,
false,
&|_| (),
)
});
let loaded_sysroot = s.spawn(|| {
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(extra_env, &targets),
))
});
let cargo_config_extra_env =
s.spawn(|| cargo_config_env(cargo_toml, extra_env, &sysroot));
thread::Result::Ok((
toolchain.join()?,
rustc_cfg.join()?,
data_layout.join()?,
rustc_dir.join()?,
loaded_sysroot.join()?,
cargo_metadata.join()?,
cargo_config_extra_env.join()?,
))
});
let (meta, error) = CargoWorkspace::fetch_metadata(
cargo_toml,
cargo_toml.parent(),
&CargoMetadataConfig {
features: config.features.clone(),
targets,
extra_args: config.extra_args.clone(),
extra_env: config.extra_env.clone(),
},
&sysroot,
false,
progress,
)
.with_context(|| {
let (
toolchain,
rustc_cfg,
data_layout,
rustc,
loaded_sysroot,
cargo_metadata,
cargo_config_extra_env,
) = match join {
Ok(it) => it,
Err(e) => std::panic::resume_unwind(e),
};
let (meta, error) = cargo_metadata.with_context(|| {
format!(
"Failed to read Cargo metadata from Cargo.toml file {cargo_toml}, {toolchain:?}",
)
})?;
let cargo_config_extra_env = cargo_config_env(cargo_toml, &config.extra_env, &sysroot);
let cargo = CargoWorkspace::new(meta, cargo_toml.clone(), cargo_config_extra_env);
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
Ok(ProjectWorkspace {
kind: ProjectWorkspaceKind::Cargo {
@ -325,33 +386,67 @@ impl ProjectWorkspace {
build_scripts: WorkspaceBuildScripts::default(),
rustc,
error: error.map(Arc::new),
set_test: config.set_test,
set_test: *set_test,
},
sysroot,
rustc_cfg,
cfg_overrides,
cfg_overrides: cfg_overrides.clone(),
toolchain,
target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
extra_includes: config.extra_includes.clone(),
extra_includes: extra_includes.clone(),
})
}
pub fn load_inline(project_json: ProjectJson, config: &CargoConfig) -> ProjectWorkspace {
pub fn load_inline(
project_json: ProjectJson,
config: &CargoConfig,
progress: &dyn Fn(String),
) -> ProjectWorkspace {
progress("Discovering sysroot".to_owned());
let mut sysroot =
Sysroot::new(project_json.sysroot.clone(), project_json.sysroot_src.clone());
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::Stitched);
let query_config = QueryConfig::Rustc(&sysroot, project_json.path().as_ref());
let toolchain = version::get(query_config, &config.extra_env).ok().flatten();
let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::Stitched);
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
tracing::info!(workspace = %project_json.manifest_or_root(), src_root = ?sysroot.src_root(), root = ?sysroot.root(), "Using sysroot");
progress("Querying project metadata".to_owned());
let query_config = QueryConfig::Rustc(&sysroot, project_json.path().as_ref());
let targets = target_tuple::get(query_config, config.target.as_deref(), &config.extra_env)
.unwrap_or_default();
// We spawn a bunch of processes to query various information about the workspace's
// toolchain and sysroot
// We can speed up loading a bit by spawning all of these processes in parallel (especially
// on systems were process spawning is delayed)
let join = thread::scope(|s| {
let toolchain =
s.spawn(|| version::get(query_config, &config.extra_env).ok().flatten());
let rustc_cfg = s.spawn(|| {
rustc_cfg::get(query_config, targets.first().map(Deref::deref), &config.extra_env)
});
let data_layout = s.spawn(|| {
target_data_layout::get(
query_config,
targets.first().map(Deref::deref),
&config.extra_env,
)
});
thread::Result::Ok((toolchain.join()?, rustc_cfg.join()?, data_layout.join()?))
});
let (toolchain, rustc_cfg, target_layout) = match join {
Ok(it) => it,
Err(e) => std::panic::resume_unwind(e),
};
let target = config.target.as_deref();
let rustc_cfg = rustc_cfg::get(query_config, target, &config.extra_env);
let data_layout = target_data_layout::get(query_config, target, &config.extra_env);
ProjectWorkspace {
kind: ProjectWorkspaceKind::Json(project_json),
sysroot,
rustc_cfg,
toolchain,
target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
target_layout: target_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
cfg_overrides: config.cfg_overrides.clone(),
extra_includes: config.extra_includes.clone(),
}
@ -374,9 +469,12 @@ impl ProjectWorkspace {
.unwrap_or_default();
let rustc_cfg = rustc_cfg::get(query_config, None, &config.extra_env);
let data_layout = target_data_layout::get(query_config, None, &config.extra_env);
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(&config.extra_env, &targets),
));
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let cargo_script = CargoWorkspace::fetch_metadata(
detached_file,

6
crates/rust-analyzer/src/cli/rustc_tests.rs
View file

@ -75,7 +75,11 @@ impl Tester {
};
let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let data_layout = target_data_layout::get(
QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
None,

1
crates/rust-analyzer/src/reload.rs
View file

@ -316,6 +316,7 @@ impl GlobalState {
let workspace = project_model::ProjectWorkspace::load_inline(
it.clone(),
&cargo_config,
&progress,
);
Ok(workspace)
}

6
crates/rust-analyzer/tests/slow-tests/main.rs
View file

@ -1152,7 +1152,11 @@ fn resolve_proc_macro() {
&AbsPathBuf::assert_utf8(std::env::current_dir().unwrap()),
&Default::default(),
);
sysroot.load_workspace(&project_model::SysrootSourceWorkspaceConfig::default_cargo());
let loaded_sysroot =
sysroot.load_workspace(&project_model::SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let proc_macro_server_path = sysroot.discover_proc_macro_srv().unwrap();