language-servers/rust-analyzer
1
0
Fork 0
mirror of https://github.com/rust-lang/rust-analyzer.git synced 2025-09-14 06:15:27 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Spawn toolchain querying processes in parallel

Browse source
This commit is contained in:
Lukas Wirth 2025-02-12 08:30:30 +01:00
parent 8aa4ae5e69
commit 8a8ff99273
6 changed files with 229 additions and 113 deletions
Download patch file Download diff file Expand all files Collapse all files

23
crates/project-model/src/sysroot.rs
View file

@ -34,14 +34,14 @@ pub struct Sysroot {
} }
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) enum SysrootWorkspace { pub enum SysrootWorkspace {
Workspace(CargoWorkspace), Workspace(CargoWorkspace),
Stitched(Stitched), Stitched(Stitched),
Empty, Empty,
} }
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
pub(crate) struct Stitched { pub struct Stitched {
crates: Arena<SysrootCrateData>, crates: Arena<SysrootCrateData>,
} }
@ -227,18 +227,21 @@ impl Sysroot {
} }
} }
pub fn load_workspace(&mut self, sysroot_source_config: &SysrootSourceWorkspaceConfig) { pub fn load_workspace(
&self,
sysroot_source_config: &SysrootSourceWorkspaceConfig,
) -> Option<SysrootWorkspace> {
assert!(matches!(self.workspace, SysrootWorkspace::Empty), "workspace already loaded"); assert!(matches!(self.workspace, SysrootWorkspace::Empty), "workspace already loaded");
let Self { root: _, src_root: Some(src_root), workspace, error: _ } = self else { return }; let Self { root: _, src_root: Some(src_root), workspace: _, error: _ } = self else {
return None;
};
if let SysrootSourceWorkspaceConfig::CargoMetadata(cargo_config) = sysroot_source_config { if let SysrootSourceWorkspaceConfig::CargoMetadata(cargo_config) = sysroot_source_config {
let library_manifest = ManifestPath::try_from(src_root.join("Cargo.toml")).unwrap(); let library_manifest = ManifestPath::try_from(src_root.join("Cargo.toml")).unwrap();
if fs::metadata(&library_manifest).is_ok() { if fs::metadata(&library_manifest).is_ok() {
if let Some(loaded) = if let Some(loaded) =
Self::load_library_via_cargo(library_manifest, src_root, cargo_config) Self::load_library_via_cargo(library_manifest, src_root, cargo_config)
{ {
*workspace = loaded; return Some(loaded);
self.load_core_check();
return;
} }
} }
} }
@ -286,11 +289,11 @@ impl Sysroot {
} }
} }
} }
*workspace = SysrootWorkspace::Stitched(stitched); Some(SysrootWorkspace::Stitched(stitched))
self.load_core_check();
} }
fn load_core_check(&mut self) { pub fn set_workspace(&mut self, workspace: SysrootWorkspace) {
self.workspace = workspace;
if self.error.is_none() { if self.error.is_none() {
if let Some(src_root) = &self.src_root { if let Some(src_root) = &self.src_root {
let has_core = match &self.workspace { let has_core = match &self.workspace {

10
crates/project-model/src/tests.rs
View file

@ -125,7 +125,10 @@ fn get_fake_sysroot() -> Sysroot {
let sysroot_dir = AbsPathBuf::assert(sysroot_path); let sysroot_dir = AbsPathBuf::assert(sysroot_path);
let sysroot_src_dir = sysroot_dir.clone(); let sysroot_src_dir = sysroot_dir.clone();
let mut sysroot = Sysroot::new(Some(sysroot_dir), Some(sysroot_src_dir)); let mut sysroot = Sysroot::new(Some(sysroot_dir), Some(sysroot_src_dir));
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo()); let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
sysroot sysroot
} }
@ -271,7 +274,10 @@ fn smoke_test_real_sysroot_cargo() {
AbsPath::assert(Utf8Path::new(env!("CARGO_MANIFEST_DIR"))), AbsPath::assert(Utf8Path::new(env!("CARGO_MANIFEST_DIR"))),
&Default::default(), &Default::default(),
); );
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo()); let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
assert!(matches!(sysroot.workspace(), SysrootWorkspace::Workspace(_))); assert!(matches!(sysroot.workspace(), SysrootWorkspace::Workspace(_)));
let project_workspace = ProjectWorkspace { let project_workspace = ProjectWorkspace {
kind: ProjectWorkspaceKind::Cargo { kind: ProjectWorkspaceKind::Cargo {

296
crates/project-model/src/workspace.rs
View file

@ -2,7 +2,7 @@
//! metadata` or `rust-project.json`) into representation stored in the salsa //! metadata` or `rust-project.json`) into representation stored in the salsa
//! database -- `CrateGraph`. //! database -- `CrateGraph`.
use std::{collections::VecDeque, fmt, fs, iter, ops::Deref, sync}; use std::{collections::VecDeque, fmt, fs, iter, ops::Deref, sync, thread};
use anyhow::Context; use anyhow::Context;
use base_db::{ use base_db::{
@ -186,7 +186,7 @@ impl ProjectWorkspace {
let project_location = project_json.parent().to_path_buf(); let project_location = project_json.parent().to_path_buf();
let project_json: ProjectJson = let project_json: ProjectJson =
ProjectJson::new(Some(project_json.clone()), &project_location, data); ProjectJson::new(Some(project_json.clone()), &project_location, data);
ProjectWorkspace::load_inline(project_json, config) ProjectWorkspace::load_inline(project_json, config, progress)
} }
ProjectManifest::CargoScript(rust_file) => { ProjectManifest::CargoScript(rust_file) => {
ProjectWorkspace::load_detached_file(rust_file, config)? ProjectWorkspace::load_detached_file(rust_file, config)?
@ -204,14 +204,28 @@ impl ProjectWorkspace {
config: &CargoConfig, config: &CargoConfig,
progress: &dyn Fn(String), progress: &dyn Fn(String),
) -> Result<ProjectWorkspace, anyhow::Error> { ) -> Result<ProjectWorkspace, anyhow::Error> {
let mut sysroot = match (&config.sysroot, &config.sysroot_src) { progress("Discovering sysroot".to_owned());
let CargoConfig {
features,
rustc_source,
extra_args,
extra_env,
set_test,
cfg_overrides,
extra_includes,
sysroot,
sysroot_src,
target,
..
} = config;
let mut sysroot = match (sysroot, sysroot_src) {
(Some(RustLibSource::Discover), None) => { (Some(RustLibSource::Discover), None) => {
Sysroot::discover(cargo_toml.parent(), &config.extra_env) Sysroot::discover(cargo_toml.parent(), extra_env)
} }
(Some(RustLibSource::Discover), Some(sysroot_src)) => { (Some(RustLibSource::Discover), Some(sysroot_src)) => {
Sysroot::discover_with_src_override( Sysroot::discover_with_src_override(
cargo_toml.parent(), cargo_toml.parent(),
&config.extra_env, extra_env,
sysroot_src.clone(), sysroot_src.clone(),
) )
} }
@ -224,100 +238,147 @@ impl ProjectWorkspace {
(None, _) => Sysroot::empty(), (None, _) => Sysroot::empty(),
}; };
let rustc_dir = match &config.rustc_source {
Some(RustLibSource::Path(path)) => ManifestPath::try_from(path.clone())
.map_err(|p| Some(format!("rustc source path is not absolute: {p}"))),
Some(RustLibSource::Discover) => sysroot
.discover_rustc_src()
.ok_or_else(|| Some("Failed to discover rustc source for sysroot.".to_owned())),
None => Err(None),
};
tracing::info!(workspace = %cargo_toml, src_root = ?sysroot.src_root(), root = ?sysroot.root(), "Using sysroot"); tracing::info!(workspace = %cargo_toml, src_root = ?sysroot.src_root(), root = ?sysroot.root(), "Using sysroot");
progress("Querying project metadata".to_owned());
let toolchain_config = QueryConfig::Cargo(&sysroot, cargo_toml); let toolchain_config = QueryConfig::Cargo(&sysroot, cargo_toml);
let targets = let targets =
target_tuple::get(toolchain_config, config.target.as_deref(), &config.extra_env) target_tuple::get(toolchain_config, target.as_deref(), extra_env).unwrap_or_default();
.unwrap_or_default();
let toolchain = version::get(toolchain_config, &config.extra_env)
.inspect_err(|e| {
tracing::error!(%e,
"failed fetching toolchain version for {cargo_toml:?} workspace"
)
})
.ok()
.flatten();
let rustc_cfg =
rustc_cfg::get(toolchain_config, targets.first().map(Deref::deref), &config.extra_env);
let cfg_overrides = config.cfg_overrides.clone();
let data_layout = target_data_layout::get(
toolchain_config,
targets.first().map(Deref::deref),
&config.extra_env,
);
if let Err(e) = &data_layout {
tracing::error!(%e, "failed fetching data layout for {cargo_toml:?} workspace");
}
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(&config.extra_env, &targets),
));
let rustc = rustc_dir.and_then(|rustc_dir| { // We spawn a bunch of processes to query various information about the workspace's
info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source"); // toolchain and sysroot
match CargoWorkspace::fetch_metadata( // We can speed up loading a bit by spawning all of these processes in parallel (especially
&rustc_dir, // on systems were process spawning is delayed)
cargo_toml.parent(), let join = thread::scope(|s| {
&CargoMetadataConfig { let workspace_dir = cargo_toml.parent();
features: crate::CargoFeatures::default(), let toolchain = s.spawn(|| {
targets: targets.clone(), version::get(toolchain_config, extra_env)
extra_args: config.extra_args.clone(), .inspect_err(|e| {
extra_env: config.extra_env.clone(), tracing::error!(%e,
}, "failed fetching toolchain version for {cargo_toml:?} workspace"
&sysroot, )
false, })
progress, .ok()
) { .flatten()
Ok((meta, _error)) => { });
let workspace = CargoWorkspace::new(meta, cargo_toml.clone(), Env::default());
let build_scripts = WorkspaceBuildScripts::rustc_crates( let rustc_cfg = s.spawn(|| {
&workspace, rustc_cfg::get(toolchain_config, targets.first().map(Deref::deref), extra_env)
cargo_toml.parent(), });
&config.extra_env, let data_layout = s.spawn(|| {
target_data_layout::get(
toolchain_config,
targets.first().map(Deref::deref),
extra_env,
).inspect_err(|e| {
tracing::error!(%e, "failed fetching data layout for {cargo_toml:?} workspace")
})
});
let rustc_dir = s.spawn(|| {
let rustc_dir = match rustc_source {
Some(RustLibSource::Path(path)) => ManifestPath::try_from(path.clone())
.map_err(|p| Some(format!("rustc source path is not absolute: {p}"))),
Some(RustLibSource::Discover) => {
sysroot.discover_rustc_src().ok_or_else(|| {
Some("Failed to discover rustc source for sysroot.".to_owned())
})
}
None => Err(None),
};
rustc_dir.and_then(|rustc_dir| {
info!(workspace = %cargo_toml, rustc_dir = %rustc_dir, "Using rustc source");
match CargoWorkspace::fetch_metadata(
&rustc_dir,
workspace_dir,
&CargoMetadataConfig {
features: crate::CargoFeatures::default(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
},
&sysroot, &sysroot,
); false,
Ok(Box::new((workspace, build_scripts))) &|_| (),
} ) {
Err(e) => { Ok((meta, _error)) => {
tracing::error!( let workspace =
%e, CargoWorkspace::new(meta, cargo_toml.clone(), Env::default());
"Failed to read Cargo metadata from rustc source at {rustc_dir}", let build_scripts = WorkspaceBuildScripts::rustc_crates(
); &workspace,
Err(Some(format!( workspace_dir,
"Failed to read Cargo metadata from rustc source at {rustc_dir}: {e}" extra_env,
))) &sysroot,
} );
} Ok(Box::new((workspace, build_scripts)))
}
Err(e) => {
tracing::error!(
%e,
"Failed to read Cargo metadata from rustc source at {rustc_dir}",
);
Err(Some(format!(
"Failed to read Cargo metadata from rustc source at {rustc_dir}: {e}"
)))
}
}
})
});
let cargo_metadata = s.spawn(|| {
CargoWorkspace::fetch_metadata(
cargo_toml,
workspace_dir,
&CargoMetadataConfig {
features: features.clone(),
targets: targets.clone(),
extra_args: extra_args.clone(),
extra_env: extra_env.clone(),
},
&sysroot,
false,
&|_| (),
)
});
let loaded_sysroot = s.spawn(|| {
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(extra_env, &targets),
))
});
let cargo_config_extra_env =
s.spawn(|| cargo_config_env(cargo_toml, extra_env, &sysroot));
thread::Result::Ok((
toolchain.join()?,
rustc_cfg.join()?,
data_layout.join()?,
rustc_dir.join()?,
loaded_sysroot.join()?,
cargo_metadata.join()?,
cargo_config_extra_env.join()?,
))
}); });
let (meta, error) = CargoWorkspace::fetch_metadata( let (
cargo_toml, toolchain,
cargo_toml.parent(), rustc_cfg,
&CargoMetadataConfig { data_layout,
features: config.features.clone(), rustc,
targets, loaded_sysroot,
extra_args: config.extra_args.clone(), cargo_metadata,
extra_env: config.extra_env.clone(), cargo_config_extra_env,
}, ) = match join {
&sysroot, Ok(it) => it,
false, Err(e) => std::panic::resume_unwind(e),
progress, };
)
.with_context(|| { let (meta, error) = cargo_metadata.with_context(|| {
format!( format!(
"Failed to read Cargo metadata from Cargo.toml file {cargo_toml}, {toolchain:?}", "Failed to read Cargo metadata from Cargo.toml file {cargo_toml}, {toolchain:?}",
) )
})?; })?;
let cargo_config_extra_env = cargo_config_env(cargo_toml, &config.extra_env, &sysroot);
let cargo = CargoWorkspace::new(meta, cargo_toml.clone(), cargo_config_extra_env); let cargo = CargoWorkspace::new(meta, cargo_toml.clone(), cargo_config_extra_env);
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
Ok(ProjectWorkspace { Ok(ProjectWorkspace {
kind: ProjectWorkspaceKind::Cargo { kind: ProjectWorkspaceKind::Cargo {
@ -325,33 +386,67 @@ impl ProjectWorkspace {
build_scripts: WorkspaceBuildScripts::default(), build_scripts: WorkspaceBuildScripts::default(),
rustc, rustc,
error: error.map(Arc::new), error: error.map(Arc::new),
set_test: config.set_test, set_test: *set_test,
}, },
sysroot, sysroot,
rustc_cfg, rustc_cfg,
cfg_overrides, cfg_overrides: cfg_overrides.clone(),
toolchain, toolchain,
target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())), target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
extra_includes: config.extra_includes.clone(), extra_includes: extra_includes.clone(),
}) })
} }
pub fn load_inline(project_json: ProjectJson, config: &CargoConfig) -> ProjectWorkspace { pub fn load_inline(
project_json: ProjectJson,
config: &CargoConfig,
progress: &dyn Fn(String),
) -> ProjectWorkspace {
progress("Discovering sysroot".to_owned());
let mut sysroot = let mut sysroot =
Sysroot::new(project_json.sysroot.clone(), project_json.sysroot_src.clone()); Sysroot::new(project_json.sysroot.clone(), project_json.sysroot_src.clone());
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::Stitched); let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::Stitched);
let query_config = QueryConfig::Rustc(&sysroot, project_json.path().as_ref()); if let Some(loaded_sysroot) = loaded_sysroot {
let toolchain = version::get(query_config, &config.extra_env).ok().flatten(); sysroot.set_workspace(loaded_sysroot);
}
tracing::info!(workspace = %project_json.manifest_or_root(), src_root = ?sysroot.src_root(), root = ?sysroot.root(), "Using sysroot");
progress("Querying project metadata".to_owned());
let query_config = QueryConfig::Rustc(&sysroot, project_json.path().as_ref());
let targets = target_tuple::get(query_config, config.target.as_deref(), &config.extra_env)
.unwrap_or_default();
// We spawn a bunch of processes to query various information about the workspace's
// toolchain and sysroot
// We can speed up loading a bit by spawning all of these processes in parallel (especially
// on systems were process spawning is delayed)
let join = thread::scope(|s| {
let toolchain =
s.spawn(|| version::get(query_config, &config.extra_env).ok().flatten());
let rustc_cfg = s.spawn(|| {
rustc_cfg::get(query_config, targets.first().map(Deref::deref), &config.extra_env)
});
let data_layout = s.spawn(|| {
target_data_layout::get(
query_config,
targets.first().map(Deref::deref),
&config.extra_env,
)
});
thread::Result::Ok((toolchain.join()?, rustc_cfg.join()?, data_layout.join()?))
});
let (toolchain, rustc_cfg, target_layout) = match join {
Ok(it) => it,
Err(e) => std::panic::resume_unwind(e),
};
let target = config.target.as_deref();
let rustc_cfg = rustc_cfg::get(query_config, target, &config.extra_env);
let data_layout = target_data_layout::get(query_config, target, &config.extra_env);
ProjectWorkspace { ProjectWorkspace {
kind: ProjectWorkspaceKind::Json(project_json), kind: ProjectWorkspaceKind::Json(project_json),
sysroot, sysroot,
rustc_cfg, rustc_cfg,
toolchain, toolchain,
target_layout: data_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())), target_layout: target_layout.map(Arc::from).map_err(|it| Arc::from(it.to_string())),
cfg_overrides: config.cfg_overrides.clone(), cfg_overrides: config.cfg_overrides.clone(),
extra_includes: config.extra_includes.clone(), extra_includes: config.extra_includes.clone(),
} }
@ -374,9 +469,12 @@ impl ProjectWorkspace {
.unwrap_or_default(); .unwrap_or_default();
let rustc_cfg = rustc_cfg::get(query_config, None, &config.extra_env); let rustc_cfg = rustc_cfg::get(query_config, None, &config.extra_env);
let data_layout = target_data_layout::get(query_config, None, &config.extra_env); let data_layout = target_data_layout::get(query_config, None, &config.extra_env);
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata( let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::CargoMetadata(
sysroot_metadata_config(&config.extra_env, &targets), sysroot_metadata_config(&config.extra_env, &targets),
)); ));
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let cargo_script = CargoWorkspace::fetch_metadata( let cargo_script = CargoWorkspace::fetch_metadata(
detached_file, detached_file,

6
crates/rust-analyzer/src/cli/rustc_tests.rs
View file

@ -75,7 +75,11 @@ impl Tester {
}; };
let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env); let mut sysroot = Sysroot::discover(tmp_file.parent().unwrap(), &cargo_config.extra_env);
sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo()); let loaded_sysroot = sysroot.load_workspace(&SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let data_layout = target_data_layout::get( let data_layout = target_data_layout::get(
QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()), QueryConfig::Rustc(&sysroot, tmp_file.parent().unwrap().as_ref()),
None, None,

1
crates/rust-analyzer/src/reload.rs
View file

@ -316,6 +316,7 @@ impl GlobalState {
let workspace = project_model::ProjectWorkspace::load_inline( let workspace = project_model::ProjectWorkspace::load_inline(
it.clone(), it.clone(),
&cargo_config, &cargo_config,
&progress,
); );
Ok(workspace) Ok(workspace)
} }

6
crates/rust-analyzer/tests/slow-tests/main.rs
View file

@ -1152,7 +1152,11 @@ fn resolve_proc_macro() {
&AbsPathBuf::assert_utf8(std::env::current_dir().unwrap()), &AbsPathBuf::assert_utf8(std::env::current_dir().unwrap()),
&Default::default(), &Default::default(),
); );
sysroot.load_workspace(&project_model::SysrootSourceWorkspaceConfig::default_cargo()); let loaded_sysroot =
sysroot.load_workspace(&project_model::SysrootSourceWorkspaceConfig::default_cargo());
if let Some(loaded_sysroot) = loaded_sysroot {
sysroot.set_workspace(loaded_sysroot);
}
let proc_macro_server_path = sysroot.discover_proc_macro_srv().unwrap(); let proc_macro_server_path = sysroot.discover_proc_macro_srv().unwrap();