language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-30 15:21:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Rebuild hosts in a separate thread and only optimize when specified

Browse source
This commit is contained in:
Brendan Hansknecht 2021-09-14 14:46:03 -07:00
parent 22722a3cf4
commit 0ef9498a69
7 changed files with 160 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

3
Cargo.lock generated
View file

@ -3483,6 +3483,7 @@ dependencies = [
"roc_editor",
"roc_fmt",
"roc_gen_llvm",
"roc_linker",
"roc_load",
"roc_module",
"roc_mono",
@ -3725,8 +3726,10 @@ dependencies = [
"iced-x86",
"memmap2 0.3.1",
"object 0.26.2",
"roc_build",
"roc_collections",
"serde",
"target-lexicon",
]
[[package]]

1
cli/Cargo.toml
View file

@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false }
roc_fmt = { path = "../compiler/fmt" }
roc_reporting = { path = "../compiler/reporting" }
roc_editor = { path = "../editor", optional = true }
roc_linker = { path = "../linker" }
# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
clap = { git = "https://github.com/rtfeldman/clap", branch = "master" }
const_format = "0.2"

72
cli/src/build.rs
View file

@ -53,6 +53,7 @@ pub fn build_file<'a>(
emit_debug_info: bool,
emit_timings: bool,
link_type: LinkType,
surgically_link: bool,
) -> Result<BuiltFile, LoadingProblem<'a>> {
let compilation_start = SystemTime::now();
let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
@ -97,7 +98,31 @@ pub fn build_file<'a>(
let host_extension = if emit_wasm { "zig" } else { "o" };
let app_extension = if emit_wasm { "bc" } else { "o" };
let cwd = roc_file_path.parent().unwrap();
let path_to_platform = loaded.platform_path.clone();
let mut host_input_path = PathBuf::from(cwd);
host_input_path.push(&*path_to_platform);
host_input_path.push("host");
host_input_path.set_extension(host_extension);
// TODO this should probably be moved before load_and_monomorphize.
// To do this we will need to preprocess files just for their exported symbols.
// Also, we should no longer need to do this once we have platforms on
// a package repository, as we can then get precompiled hosts from there.
let rebuild_thread = spawn_rebuild_thread(
opt_level,
surgically_link,
host_input_path.clone(),
target.clone(),
loaded
.exposed_to_host
.keys()
.map(|x| x.as_str(&loaded.interns).to_string())
.collect(),
);
// TODO try to move as much of this linking as possible to the precompiled
// host, to minimize the amount of host-application linking required.
let app_o_file = Builder::new()
.prefix("roc_app")
.suffix(&format!(".{}", app_extension))
@ -154,7 +179,6 @@ pub fn build_file<'a>(
program::report_problems(&mut loaded);
let loaded = loaded;
let cwd = roc_file_path.parent().unwrap();
let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows
let code_gen_timing = program::gen_from_mono_module(
arena,
@ -198,28 +222,15 @@ pub fn build_file<'a>(
);
}
// Step 2: link the precompiled host and compiled app
let mut host_input_path = PathBuf::from(cwd);
host_input_path.push(&*path_to_platform);
host_input_path.push("host");
host_input_path.set_extension(host_extension);
// TODO we should no longer need to do this once we have platforms on
// a package repository, as we can then get precompiled hosts from there.
let rebuild_host_start = SystemTime::now();
rebuild_host(target, host_input_path.as_path());
let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
let rebuild_duration = rebuild_thread.join().unwrap();
if emit_timings {
println!(
"Finished rebuilding the host in {} ms\n",
rebuild_host_end.as_millis()
"Finished rebuilding and preprocessing the host in {} ms\n",
rebuild_duration
);
}
// TODO try to move as much of this linking as possible to the precompiled
// host, to minimize the amount of host-application linking required.
// Step 2: link the precompiled host and compiled app
let link_start = SystemTime::now();
let (mut child, binary_path) = // TODO use lld
link(
@ -260,3 +271,28 @@ pub fn build_file<'a>(
total_time,
})
}
fn spawn_rebuild_thread(
opt_level: OptLevel,
surgically_link: bool,
host_input_path: PathBuf,
target: Triple,
exported_symbols: Vec<String>,
) -> std::thread::JoinHandle<u128> {
let thread_local_target = target.clone();
std::thread::spawn(move || {
let rebuild_host_start = SystemTime::now();
if surgically_link {
roc_linker::build_and_preprocess_host(
&thread_local_target,
host_input_path.as_path(),
exported_symbols,
)
.unwrap();
} else {
rebuild_host(opt_level, &thread_local_target, host_input_path.as_path());
}
let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
rebuild_host_end.as_millis()
})
}

21
cli/src/lib.rs
View file

@ -32,6 +32,7 @@ pub const FLAG_OPTIMIZE: &str = "optimize";
pub const FLAG_LIB: &str = "lib";
pub const FLAG_BACKEND: &str = "backend";
pub const FLAG_TIME: &str = "time";
pub const FLAG_LINK: &str = "roc-linker";
pub const ROC_FILE: &str = "ROC_FILE";
pub const BACKEND: &str = "BACKEND";
pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES";
@ -81,6 +82,12 @@ pub fn build_app<'a>() -> App<'a> {
.help("Prints detailed compilation time information.")
.required(false),
)
.arg(
Arg::with_name(FLAG_LINK)
.long(FLAG_LINK)
.help("Uses the roc linker instead of the system linker.")
.required(false),
)
)
.subcommand(App::new(CMD_RUN)
.about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`")
@ -143,6 +150,12 @@ pub fn build_app<'a>() -> App<'a> {
.help("Prints detailed compilation time information.")
.required(false),
)
.arg(
Arg::with_name(FLAG_LINK)
.long(FLAG_LINK)
.help("Uses the roc linker instead of the system linker.")
.required(false),
)
.arg(
Arg::with_name(FLAG_BACKEND)
.long(FLAG_BACKEND)
@ -223,6 +236,13 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
} else {
LinkType::Executable
};
let surgically_link = matches.is_present(FLAG_LINK);
if surgically_link && !roc_linker::supported(&link_type, &target) {
panic!(
"Link type, {:?}, with target, {}, not supported by roc linker",
link_type, target
);
}
let path = Path::new(filename);
@ -255,6 +275,7 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
emit_debug_info,
emit_timings,
link_type,
surgically_link,
);
match res_binary_path {

87
compiler/build/src/link.rs
View file

@ -83,8 +83,10 @@ pub fn build_zig_host_native(
zig_host_src: &str,
zig_str_path: &str,
target: &str,
opt_level: OptLevel,
) -> Output {
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", env_path)
.env("HOME", env_home)
@ -102,14 +104,14 @@ pub fn build_zig_host_native(
"--library",
"c",
"-fPIC",
"-O",
"ReleaseSafe",
// cross-compile?
"-target",
target,
])
.output()
.unwrap()
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
#[cfg(target_os = "macos")]
@ -120,6 +122,7 @@ pub fn build_zig_host_native(
zig_host_src: &str,
zig_str_path: &str,
_target: &str,
opt_level: OptLevel,
) -> Output {
use serde_json::Value;
@ -161,7 +164,8 @@ pub fn build_zig_host_native(
zig_compiler_rt_path.push("special");
zig_compiler_rt_path.push("compiler_rt.zig");
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
@ -182,11 +186,11 @@ pub fn build_zig_host_native(
"--library",
"c",
"-fPIC",
"-O",
"ReleaseSafe",
])
.output()
.unwrap()
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
pub fn build_zig_host_wasm32(
@ -195,6 +199,7 @@ pub fn build_zig_host_wasm32(
emit_bin: &str,
zig_host_src: &str,
zig_str_path: &str,
opt_level: OptLevel,
) -> Output {
// NOTE currently just to get compiler warnings if the host code is invalid.
// the produced artifact is not used
@ -204,7 +209,8 @@ pub fn build_zig_host_wasm32(
// we'd like to compile with `-target wasm32-wasi` but that is blocked on
//
// https://github.com/ziglang/zig/issues/9414
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", env_path)
.env("HOME", env_home)
@ -226,14 +232,14 @@ pub fn build_zig_host_wasm32(
// "wasm32-wasi",
// "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll",
"-fPIC",
"-O",
"ReleaseSafe",
])
.output()
.unwrap()
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) {
let c_host_src = host_input_path.with_file_name("host.c");
let c_host_dest = host_input_path.with_file_name("c_host.o");
let zig_host_src = host_input_path.with_file_name("host.zig");
@ -266,6 +272,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
&emit_bin,
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
opt_level,
)
}
Architecture::X86_64 => {
@ -277,6 +284,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
"native",
opt_level,
)
}
Architecture::X86_32(_) => {
@ -288,6 +296,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
"i386-linux-musl",
opt_level,
)
}
_ => panic!("Unsupported architecture {:?}", target.architecture),
@ -296,19 +305,18 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
validate_output("host.zig", "zig", output)
} else {
// Compile host.c
let output = Command::new("clang")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-O2",
let mut command = Command::new("clang");
command.env_clear().env("PATH", &env_path).args(&[
"-fPIC",
"-c",
c_host_src.to_str().unwrap(),
"-o",
c_host_dest.to_str().unwrap(),
])
.output()
.unwrap();
]);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
let output = command.output().unwrap();
validate_output("host.c", "clang", output);
}
@ -318,13 +326,14 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
let cargo_dir = host_input_path.parent().unwrap();
let libhost_dir = cargo_dir.join("target").join("release");
let output = Command::new("cargo")
.args(&["build", "--release"])
.current_dir(cargo_dir)
.output()
.unwrap();
let mut command = Command::new("cargo");
command.arg("build").current_dir(cargo_dir);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("--release");
}
let output = command.output().unwrap();
validate_output("src/lib.rs", "cargo build --release", output);
validate_output("src/lib.rs", "cargo build", output);
let output = Command::new("ld")
.env_clear()
@ -344,14 +353,16 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
validate_output("c_host.o", "ld", output);
} else if rust_host_src.exists() {
// Compile and link host.rs, if it exists
let output = Command::new("rustc")
.args(&[
let mut command = Command::new("rustc");
command.args(&[
rust_host_src.to_str().unwrap(),
"-o",
rust_host_dest.to_str().unwrap(),
])
.output()
.unwrap();
]);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O");
}
let output = command.output().unwrap();
validate_output("host.rs", "rustc", output);

2
linker/Cargo.toml
View file

@ -18,6 +18,7 @@ test = false
bench = false
[dependencies]
roc_build = { path = "../compiler/build", default-features = false }
roc_collections = { path = "../compiler/collections" }
bumpalo = { version = "3.6", features = ["collections"] }
# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
@ -27,3 +28,4 @@ memmap2 = "0.3"
object = { version = "0.26", features = ["read"] }
serde = { version = "1.0", features = ["derive"] }
bincode = "1.3"
target-lexicon = "0.12.2"

22
linker/src/lib.rs
View file

@ -8,6 +8,7 @@ use object::{
Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
Symbol, SymbolIndex, SymbolSection,
};
use roc_build::link::LinkType;
use roc_collections::all::MutMap;
use std::cmp::Ordering;
use std::convert::TryFrom;
@ -19,6 +20,7 @@ use std::mem;
use std::os::raw::c_char;
use std::path::Path;
use std::time::{Duration, SystemTime};
use target_lexicon::Triple;
mod metadata;
@ -122,6 +124,26 @@ pub fn build_app<'a>() -> App<'a> {
)
}
pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
link_type == &LinkType::Executable
&& target.architecture == target_lexicon::Architecture::X86_64
&& target.operating_system == target_lexicon::OperatingSystem::Linux
&& target.binary_format == target_lexicon::BinaryFormat::Elf
}
pub fn build_and_preprocess_host(
target: &Triple,
host_input_path: &Path,
exposed_to_host: Vec<String>,
) -> io::Result<()> {
let lib = generate_dynamic_lib(exposed_to_host)?;
Ok(())
}
fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<()> {
Ok(())
}
// TODO: Most of this file is a mess of giant functions just to check if things work.
// Clean it all up and refactor nicely.
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {