From 0ef9498a6945fc936b30ad9ed71108ea7bf1fd74 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 14 Sep 2021 14:46:03 -0700 Subject: [PATCH] Rebuild hosts in a separate thread and only optimize when specified --- Cargo.lock | 3 ++ cli/Cargo.toml | 1 + cli/src/build.rs | 72 +++++++++++++++++++------- cli/src/lib.rs | 21 ++++++++ compiler/build/src/link.rs | 103 ++++++++++++++++++++----------------- linker/Cargo.toml | 2 + linker/src/lib.rs | 22 ++++++++ 7 files changed, 160 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b6373852f..6819b9a48d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3483,6 +3483,7 @@ dependencies = [ "roc_editor", "roc_fmt", "roc_gen_llvm", + "roc_linker", "roc_load", "roc_module", "roc_mono", @@ -3725,8 +3726,10 @@ dependencies = [ "iced-x86", "memmap2 0.3.1", "object 0.26.2", + "roc_build", "roc_collections", "serde", + "target-lexicon", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0225a11916..97ecb81a46 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false } roc_fmt = { path = "../compiler/fmt" } roc_reporting = { path = "../compiler/reporting" } roc_editor = { path = "../editor", optional = true } +roc_linker = { path = "../linker" } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } const_format = "0.2" diff --git a/cli/src/build.rs b/cli/src/build.rs index face340a56..3426bdad55 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -53,6 +53,7 @@ pub fn build_file<'a>( emit_debug_info: bool, emit_timings: bool, link_type: LinkType, + surgically_link: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -97,7 +98,31 @@ pub fn build_file<'a>( let host_extension = if emit_wasm { "zig" } else { "o" }; let app_extension = if emit_wasm { "bc" } else { "o" }; + let cwd = roc_file_path.parent().unwrap(); let path_to_platform = loaded.platform_path.clone(); + let mut host_input_path = PathBuf::from(cwd); + host_input_path.push(&*path_to_platform); + host_input_path.push("host"); + host_input_path.set_extension(host_extension); + + // TODO this should probably be moved before load_and_monomorphize. + // To do this we will need to preprocess files just for their exported symbols. + // Also, we should no longer need to do this once we have platforms on + // a package repository, as we can then get precompiled hosts from there. + let rebuild_thread = spawn_rebuild_thread( + opt_level, + surgically_link, + host_input_path.clone(), + target.clone(), + loaded + .exposed_to_host + .keys() + .map(|x| x.as_str(&loaded.interns).to_string()) + .collect(), + ); + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. let app_o_file = Builder::new() .prefix("roc_app") .suffix(&format!(".{}", app_extension)) @@ -154,7 +179,6 @@ pub fn build_file<'a>( program::report_problems(&mut loaded); let loaded = loaded; - let cwd = roc_file_path.parent().unwrap(); let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows let code_gen_timing = program::gen_from_mono_module( arena, @@ -198,28 +222,15 @@ pub fn build_file<'a>( ); } - // Step 2: link the precompiled host and compiled app - let mut host_input_path = PathBuf::from(cwd); - - host_input_path.push(&*path_to_platform); - host_input_path.push("host"); - host_input_path.set_extension(host_extension); - - // TODO we should no longer need to do this once we have platforms on - // a package repository, as we can then get precompiled hosts from there. - let rebuild_host_start = SystemTime::now(); - rebuild_host(target, host_input_path.as_path()); - let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); - + let rebuild_duration = rebuild_thread.join().unwrap(); if emit_timings { println!( - "Finished rebuilding the host in {} ms\n", - rebuild_host_end.as_millis() + "Finished rebuilding and preprocessing the host in {} ms\n", + rebuild_duration ); } - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. + // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); let (mut child, binary_path) = // TODO use lld link( @@ -260,3 +271,28 @@ pub fn build_file<'a>( total_time, }) } + +fn spawn_rebuild_thread( + opt_level: OptLevel, + surgically_link: bool, + host_input_path: PathBuf, + target: Triple, + exported_symbols: Vec, +) -> std::thread::JoinHandle { + let thread_local_target = target.clone(); + std::thread::spawn(move || { + let rebuild_host_start = SystemTime::now(); + if surgically_link { + roc_linker::build_and_preprocess_host( + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host(opt_level, &thread_local_target, host_input_path.as_path()); + } + let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); + rebuild_host_end.as_millis() + }) +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 2027e2f300..d73c38054f 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -32,6 +32,7 @@ pub const FLAG_OPTIMIZE: &str = "optimize"; pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; pub const FLAG_TIME: &str = "time"; +pub const FLAG_LINK: &str = "roc-linker"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -81,6 +82,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -143,6 +150,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -223,6 +236,13 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { } else { LinkType::Executable }; + let surgically_link = matches.is_present(FLAG_LINK); + if surgically_link && !roc_linker::supported(&link_type, &target) { + panic!( + "Link type, {:?}, with target, {}, not supported by roc linker", + link_type, target + ); + } let path = Path::new(filename); @@ -255,6 +275,7 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { emit_debug_info, emit_timings, link_type, + surgically_link, ); match res_binary_path { diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index d3f7aba399..5e1de46012 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -83,8 +83,10 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, target: &str, + opt_level: OptLevel, ) -> Output { - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -102,14 +104,14 @@ pub fn build_zig_host_native( "--library", "c", "-fPIC", - "-O", - "ReleaseSafe", // cross-compile? "-target", target, - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } #[cfg(target_os = "macos")] @@ -120,6 +122,7 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, _target: &str, + opt_level: OptLevel, ) -> Output { use serde_json::Value; @@ -161,7 +164,8 @@ pub fn build_zig_host_native( zig_compiler_rt_path.push("special"); zig_compiler_rt_path.push("compiler_rt.zig"); - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", &env_path) .env("HOME", &env_home) @@ -182,11 +186,11 @@ pub fn build_zig_host_native( "--library", "c", "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } pub fn build_zig_host_wasm32( @@ -195,6 +199,7 @@ pub fn build_zig_host_wasm32( emit_bin: &str, zig_host_src: &str, zig_str_path: &str, + opt_level: OptLevel, ) -> Output { // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used @@ -204,7 +209,8 @@ pub fn build_zig_host_wasm32( // we'd like to compile with `-target wasm32-wasi` but that is blocked on // // https://github.com/ziglang/zig/issues/9414 - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -226,14 +232,14 @@ pub fn build_zig_host_wasm32( // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } -pub fn rebuild_host(target: &Triple, host_input_path: &Path) { +pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); @@ -266,6 +272,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { &emit_bin, zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), + opt_level, ) } Architecture::X86_64 => { @@ -277,6 +284,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "native", + opt_level, ) } Architecture::X86_32(_) => { @@ -288,6 +296,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "i386-linux-musl", + opt_level, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), @@ -296,19 +305,18 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { validate_output("host.zig", "zig", output) } else { // Compile host.c - let output = Command::new("clang") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-O2", - "-fPIC", - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("clang"); + command.env_clear().env("PATH", &env_path).args(&[ + "-fPIC", + "-c", + c_host_src.to_str().unwrap(), + "-o", + c_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + let output = command.output().unwrap(); validate_output("host.c", "clang", output); } @@ -318,13 +326,14 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { let cargo_dir = host_input_path.parent().unwrap(); let libhost_dir = cargo_dir.join("target").join("release"); - let output = Command::new("cargo") - .args(&["build", "--release"]) - .current_dir(cargo_dir) - .output() - .unwrap(); + let mut command = Command::new("cargo"); + command.arg("build").current_dir(cargo_dir); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("--release"); + } + let output = command.output().unwrap(); - validate_output("src/lib.rs", "cargo build --release", output); + validate_output("src/lib.rs", "cargo build", output); let output = Command::new("ld") .env_clear() @@ -344,14 +353,16 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { validate_output("c_host.o", "ld", output); } else if rust_host_src.exists() { // Compile and link host.rs, if it exists - let output = Command::new("rustc") - .args(&[ - rust_host_src.to_str().unwrap(), - "-o", - rust_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("rustc"); + command.args(&[ + rust_host_src.to_str().unwrap(), + "-o", + rust_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O"); + } + let output = command.output().unwrap(); validate_output("host.rs", "rustc", output); diff --git a/linker/Cargo.toml b/linker/Cargo.toml index 43461fc276..520ed8f660 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -18,6 +18,7 @@ test = false bench = false [dependencies] +roc_build = { path = "../compiler/build", default-features = false } roc_collections = { path = "../compiler/collections" } bumpalo = { version = "3.6", features = ["collections"] } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it @@ -27,3 +28,4 @@ memmap2 = "0.3" object = { version = "0.26", features = ["read"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" +target-lexicon = "0.12.2" diff --git a/linker/src/lib.rs b/linker/src/lib.rs index e5e6b6f1a8..c747d184f8 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -8,6 +8,7 @@ use object::{ Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, Symbol, SymbolIndex, SymbolSection, }; +use roc_build::link::LinkType; use roc_collections::all::MutMap; use std::cmp::Ordering; use std::convert::TryFrom; @@ -19,6 +20,7 @@ use std::mem; use std::os::raw::c_char; use std::path::Path; use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; mod metadata; @@ -122,6 +124,26 @@ pub fn build_app<'a>() -> App<'a> { ) } +pub fn supported(link_type: &LinkType, target: &Triple) -> bool { + link_type == &LinkType::Executable + && target.architecture == target_lexicon::Architecture::X86_64 + && target.operating_system == target_lexicon::OperatingSystem::Linux + && target.binary_format == target_lexicon::BinaryFormat::Elf +} + +pub fn build_and_preprocess_host( + target: &Triple, + host_input_path: &Path, + exposed_to_host: Vec, +) -> io::Result<()> { + let lib = generate_dynamic_lib(exposed_to_host)?; + Ok(()) +} + +fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result<()> { + Ok(()) +} + // TODO: Most of this file is a mess of giant functions just to check if things work. // Clean it all up and refactor nicely. pub fn preprocess(matches: &ArgMatches) -> io::Result {