From 0ef9498a6945fc936b30ad9ed71108ea7bf1fd74 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 14 Sep 2021 14:46:03 -0700 Subject: [PATCH 01/16] Rebuild hosts in a separate thread and only optimize when specified --- Cargo.lock | 3 ++ cli/Cargo.toml | 1 + cli/src/build.rs | 72 +++++++++++++++++++------- cli/src/lib.rs | 21 ++++++++ compiler/build/src/link.rs | 103 ++++++++++++++++++++----------------- linker/Cargo.toml | 2 + linker/src/lib.rs | 22 ++++++++ 7 files changed, 160 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b6373852f..6819b9a48d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3483,6 +3483,7 @@ dependencies = [ "roc_editor", "roc_fmt", "roc_gen_llvm", + "roc_linker", "roc_load", "roc_module", "roc_mono", @@ -3725,8 +3726,10 @@ dependencies = [ "iced-x86", "memmap2 0.3.1", "object 0.26.2", + "roc_build", "roc_collections", "serde", + "target-lexicon", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0225a11916..97ecb81a46 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false } roc_fmt = { path = "../compiler/fmt" } roc_reporting = { path = "../compiler/reporting" } roc_editor = { path = "../editor", optional = true } +roc_linker = { path = "../linker" } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } const_format = "0.2" diff --git a/cli/src/build.rs b/cli/src/build.rs index face340a56..3426bdad55 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -53,6 +53,7 @@ pub fn build_file<'a>( emit_debug_info: bool, emit_timings: bool, link_type: LinkType, + surgically_link: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -97,7 +98,31 @@ pub fn build_file<'a>( let host_extension = if emit_wasm { "zig" } else { "o" }; let app_extension = if emit_wasm { "bc" } else { "o" }; + let cwd = roc_file_path.parent().unwrap(); let path_to_platform = loaded.platform_path.clone(); + let mut host_input_path = PathBuf::from(cwd); + host_input_path.push(&*path_to_platform); + host_input_path.push("host"); + host_input_path.set_extension(host_extension); + + // TODO this should probably be moved before load_and_monomorphize. + // To do this we will need to preprocess files just for their exported symbols. + // Also, we should no longer need to do this once we have platforms on + // a package repository, as we can then get precompiled hosts from there. + let rebuild_thread = spawn_rebuild_thread( + opt_level, + surgically_link, + host_input_path.clone(), + target.clone(), + loaded + .exposed_to_host + .keys() + .map(|x| x.as_str(&loaded.interns).to_string()) + .collect(), + ); + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. let app_o_file = Builder::new() .prefix("roc_app") .suffix(&format!(".{}", app_extension)) @@ -154,7 +179,6 @@ pub fn build_file<'a>( program::report_problems(&mut loaded); let loaded = loaded; - let cwd = roc_file_path.parent().unwrap(); let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows let code_gen_timing = program::gen_from_mono_module( arena, @@ -198,28 +222,15 @@ pub fn build_file<'a>( ); } - // Step 2: link the precompiled host and compiled app - let mut host_input_path = PathBuf::from(cwd); - - host_input_path.push(&*path_to_platform); - host_input_path.push("host"); - host_input_path.set_extension(host_extension); - - // TODO we should no longer need to do this once we have platforms on - // a package repository, as we can then get precompiled hosts from there. - let rebuild_host_start = SystemTime::now(); - rebuild_host(target, host_input_path.as_path()); - let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); - + let rebuild_duration = rebuild_thread.join().unwrap(); if emit_timings { println!( - "Finished rebuilding the host in {} ms\n", - rebuild_host_end.as_millis() + "Finished rebuilding and preprocessing the host in {} ms\n", + rebuild_duration ); } - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. + // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); let (mut child, binary_path) = // TODO use lld link( @@ -260,3 +271,28 @@ pub fn build_file<'a>( total_time, }) } + +fn spawn_rebuild_thread( + opt_level: OptLevel, + surgically_link: bool, + host_input_path: PathBuf, + target: Triple, + exported_symbols: Vec, +) -> std::thread::JoinHandle { + let thread_local_target = target.clone(); + std::thread::spawn(move || { + let rebuild_host_start = SystemTime::now(); + if surgically_link { + roc_linker::build_and_preprocess_host( + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host(opt_level, &thread_local_target, host_input_path.as_path()); + } + let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); + rebuild_host_end.as_millis() + }) +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 2027e2f300..d73c38054f 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -32,6 +32,7 @@ pub const FLAG_OPTIMIZE: &str = "optimize"; pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; pub const FLAG_TIME: &str = "time"; +pub const FLAG_LINK: &str = "roc-linker"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -81,6 +82,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -143,6 +150,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -223,6 +236,13 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { } else { LinkType::Executable }; + let surgically_link = matches.is_present(FLAG_LINK); + if surgically_link && !roc_linker::supported(&link_type, &target) { + panic!( + "Link type, {:?}, with target, {}, not supported by roc linker", + link_type, target + ); + } let path = Path::new(filename); @@ -255,6 +275,7 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { emit_debug_info, emit_timings, link_type, + surgically_link, ); match res_binary_path { diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index d3f7aba399..5e1de46012 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -83,8 +83,10 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, target: &str, + opt_level: OptLevel, ) -> Output { - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -102,14 +104,14 @@ pub fn build_zig_host_native( "--library", "c", "-fPIC", - "-O", - "ReleaseSafe", // cross-compile? "-target", target, - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } #[cfg(target_os = "macos")] @@ -120,6 +122,7 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, _target: &str, + opt_level: OptLevel, ) -> Output { use serde_json::Value; @@ -161,7 +164,8 @@ pub fn build_zig_host_native( zig_compiler_rt_path.push("special"); zig_compiler_rt_path.push("compiler_rt.zig"); - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", &env_path) .env("HOME", &env_home) @@ -182,11 +186,11 @@ pub fn build_zig_host_native( "--library", "c", "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } pub fn build_zig_host_wasm32( @@ -195,6 +199,7 @@ pub fn build_zig_host_wasm32( emit_bin: &str, zig_host_src: &str, zig_str_path: &str, + opt_level: OptLevel, ) -> Output { // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used @@ -204,7 +209,8 @@ pub fn build_zig_host_wasm32( // we'd like to compile with `-target wasm32-wasi` but that is blocked on // // https://github.com/ziglang/zig/issues/9414 - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -226,14 +232,14 @@ pub fn build_zig_host_wasm32( // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } -pub fn rebuild_host(target: &Triple, host_input_path: &Path) { +pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); @@ -266,6 +272,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { &emit_bin, zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), + opt_level, ) } Architecture::X86_64 => { @@ -277,6 +284,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "native", + opt_level, ) } Architecture::X86_32(_) => { @@ -288,6 +296,7 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "i386-linux-musl", + opt_level, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), @@ -296,19 +305,18 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { validate_output("host.zig", "zig", output) } else { // Compile host.c - let output = Command::new("clang") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-O2", - "-fPIC", - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("clang"); + command.env_clear().env("PATH", &env_path).args(&[ + "-fPIC", + "-c", + c_host_src.to_str().unwrap(), + "-o", + c_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + let output = command.output().unwrap(); validate_output("host.c", "clang", output); } @@ -318,13 +326,14 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { let cargo_dir = host_input_path.parent().unwrap(); let libhost_dir = cargo_dir.join("target").join("release"); - let output = Command::new("cargo") - .args(&["build", "--release"]) - .current_dir(cargo_dir) - .output() - .unwrap(); + let mut command = Command::new("cargo"); + command.arg("build").current_dir(cargo_dir); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("--release"); + } + let output = command.output().unwrap(); - validate_output("src/lib.rs", "cargo build --release", output); + validate_output("src/lib.rs", "cargo build", output); let output = Command::new("ld") .env_clear() @@ -344,14 +353,16 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { validate_output("c_host.o", "ld", output); } else if rust_host_src.exists() { // Compile and link host.rs, if it exists - let output = Command::new("rustc") - .args(&[ - rust_host_src.to_str().unwrap(), - "-o", - rust_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("rustc"); + command.args(&[ + rust_host_src.to_str().unwrap(), + "-o", + rust_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O"); + } + let output = command.output().unwrap(); validate_output("host.rs", "rustc", output); diff --git a/linker/Cargo.toml b/linker/Cargo.toml index 43461fc276..520ed8f660 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -18,6 +18,7 @@ test = false bench = false [dependencies] +roc_build = { path = "../compiler/build", default-features = false } roc_collections = { path = "../compiler/collections" } bumpalo = { version = "3.6", features = ["collections"] } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it @@ -27,3 +28,4 @@ memmap2 = "0.3" object = { version = "0.26", features = ["read"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" +target-lexicon = "0.12.2" diff --git a/linker/src/lib.rs b/linker/src/lib.rs index e5e6b6f1a8..c747d184f8 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -8,6 +8,7 @@ use object::{ Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, Symbol, SymbolIndex, SymbolSection, }; +use roc_build::link::LinkType; use roc_collections::all::MutMap; use std::cmp::Ordering; use std::convert::TryFrom; @@ -19,6 +20,7 @@ use std::mem; use std::os::raw::c_char; use std::path::Path; use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; mod metadata; @@ -122,6 +124,26 @@ pub fn build_app<'a>() -> App<'a> { ) } +pub fn supported(link_type: &LinkType, target: &Triple) -> bool { + link_type == &LinkType::Executable + && target.architecture == target_lexicon::Architecture::X86_64 + && target.operating_system == target_lexicon::OperatingSystem::Linux + && target.binary_format == target_lexicon::BinaryFormat::Elf +} + +pub fn build_and_preprocess_host( + target: &Triple, + host_input_path: &Path, + exposed_to_host: Vec, +) -> io::Result<()> { + let lib = generate_dynamic_lib(exposed_to_host)?; + Ok(()) +} + +fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result<()> { + Ok(()) +} + // TODO: Most of this file is a mess of giant functions just to check if things work. // Clean it all up and refactor nicely. pub fn preprocess(matches: &ArgMatches) -> io::Result { From 7297e969bd95063233c14438935a82270d7b0db4 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 14 Sep 2021 17:30:26 -0700 Subject: [PATCH 02/16] Fix cargo debug build --- compiler/build/src/link.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 5e1de46012..08dd7f0f6e 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -324,7 +324,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); - let libhost_dir = cargo_dir.join("target").join("release"); + let libhost_dir = + cargo_dir + .join("target") + .join(if matches!(opt_level, OptLevel::Optimize) { + "release" + } else { + "debug" + }); let mut command = Command::new("cargo"); command.arg("build").current_dir(cargo_dir); From e96291e9a7b149174eb1fef1ff0297eb60714a77 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 14 Sep 2021 21:59:15 -0700 Subject: [PATCH 03/16] Enable rebuilding hosts into a dynamic executable --- Cargo.lock | 2 + cli/src/build.rs | 8 +- compiler/build/src/link.rs | 293 +++++++++++++++++++++++++------------ linker/Cargo.toml | 2 + linker/src/lib.rs | 67 +++++++-- 5 files changed, 260 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6819b9a48d..c17cb03261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,8 +3728,10 @@ dependencies = [ "object 0.26.2", "roc_build", "roc_collections", + "roc_mono", "serde", "target-lexicon", + "tempfile", ] [[package]] diff --git a/cli/src/build.rs b/cli/src/build.rs index 3426bdad55..fec21f420b 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -284,13 +284,19 @@ fn spawn_rebuild_thread( let rebuild_host_start = SystemTime::now(); if surgically_link { roc_linker::build_and_preprocess_host( + opt_level, &thread_local_target, host_input_path.as_path(), exported_symbols, ) .unwrap(); } else { - rebuild_host(opt_level, &thread_local_target, host_input_path.as_path()); + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); } let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); rebuild_host_end.as_millis() diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 08dd7f0f6e..d6de4b12d9 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -84,30 +84,35 @@ pub fn build_zig_host_native( zig_str_path: &str, target: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { let mut command = Command::new("zig"); command .env_clear() .env("PATH", env_path) - .env("HOME", env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "-fcompiler-rt", - // include libc - "--library", - "c", - "-fPIC", - // cross-compile? - "-target", - target, - ]); + .env("HOME", env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + } else { + command.arg("build-obj"); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "-fcompiler-rt", + // include libc + "--library", + "c", + "-fPIC", + // cross-compile? + "-target", + target, + ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); } @@ -123,6 +128,7 @@ pub fn build_zig_host_native( zig_str_path: &str, _target: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { use serde_json::Value; @@ -168,25 +174,29 @@ pub fn build_zig_host_native( command .env_clear() .env("PATH", &env_path) - .env("HOME", &env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "--pkg-begin", - "compiler_rt", - zig_compiler_rt_path.to_str().unwrap(), - "--pkg-end", - // include libc - "--library", - "c", - "-fPIC", - ]); + .env("HOME", &env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + } else { + command.arg("build-obj"); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "--pkg-begin", + "compiler_rt", + zig_compiler_rt_path.to_str().unwrap(), + "--pkg-end", + // include libc + "--library", + "c", + "-fPIC", + ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); } @@ -200,7 +210,11 @@ pub fn build_zig_host_wasm32( zig_host_src: &str, zig_str_path: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { + if shared_lib_path.is_some() { + unimplemented!("Linking a shared library to wasm not yet implemented"); + } // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used // @@ -239,14 +253,56 @@ pub fn build_zig_host_wasm32( command.output().unwrap() } -pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) { +pub fn build_c_host_native( + env_path: &str, + env_home: &str, + dest: &str, + sources: &[&str], + opt_level: OptLevel, + shared_lib_path: Option<&Path>, +) -> Output { + let mut command = Command::new("clang"); + command + .env_clear() + .env("PATH", &env_path) + .env("HOME", &env_home) + .args(sources) + .args(&["-fPIC", "-o", dest]); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + shared_lib_path.to_str().unwrap(), + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + ]); + } else { + command.arg("-c"); + } + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + command.output().unwrap() +} + +pub fn rebuild_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + shared_lib_path: Option<&Path>, +) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); - let host_dest_native = host_input_path.with_file_name("host.o"); + let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() { + "dynhost" + } else { + "host.o" + }); let host_dest_wasm = host_input_path.with_file_name("host.bc"); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); @@ -273,6 +329,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), opt_level, + shared_lib_path, ) } Architecture::X86_64 => { @@ -285,6 +342,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_str_path.to_str().unwrap(), "native", opt_level, + shared_lib_path, ) } Architecture::X86_32(_) => { @@ -297,31 +355,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_str_path.to_str().unwrap(), "i386-linux-musl", opt_level, + shared_lib_path, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), }; validate_output("host.zig", "zig", output) - } else { - // Compile host.c - let mut command = Command::new("clang"); - command.env_clear().env("PATH", &env_path).args(&[ - "-fPIC", - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]); - if matches!(opt_level, OptLevel::Optimize) { - command.arg("-O2"); - } - let output = command.output().unwrap(); - - validate_output("host.c", "clang", output); - } - - if cargo_host_src.exists() { + } else if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); let libhost_dir = @@ -332,6 +373,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path } else { "debug" }); + let libhost = libhost_dir.join("libhost.a"); let mut command = Command::new("cargo"); command.arg("build").current_dir(cargo_dir); @@ -342,22 +384,54 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path validate_output("src/lib.rs", "cargo build", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - "-L", - libhost_dir.to_str().unwrap(), - c_host_dest.to_str().unwrap(), - "-lhost", - "-o", + // Cargo hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); - validate_output("c_host.o", "ld", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + "-L", + libhost_dir.to_str().unwrap(), + c_host_dest.to_str().unwrap(), + "-lhost", + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); + validate_output("c_host.o", "ld", output); + + // Clean up c_host.o + let output = Command::new("rm") + .env_clear() + .args(&["-f", c_host_dest.to_str().unwrap()]) + .output() + .unwrap(); + + validate_output("rust_host.o", "rm", output); + } } else if rust_host_src.exists() { // Compile and link host.rs, if it exists let mut command = Command::new("rustc"); @@ -373,22 +447,49 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path validate_output("host.rs", "rustc", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - c_host_dest.to_str().unwrap(), - rust_host_dest.to_str().unwrap(), - "-o", + // Rust hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[ + c_host_src.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + ], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); - validate_output("rust_host.o", "ld", output); + validate_output("host.c", "clang", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + c_host_dest.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); - // Clean up rust_host.o + validate_output("rust_host.o", "ld", output); + } + + // Clean up rust_host.o and c_host.o let output = Command::new("rm") .env_clear() .args(&[ @@ -400,15 +501,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path .unwrap(); validate_output("rust_host.o", "rm", output); - } else if c_host_dest.exists() { - // Clean up c_host.o - let output = Command::new("mv") - .env_clear() - .args(&[c_host_dest, host_dest_native]) - .output() - .unwrap(); - - validate_output("c_host.o", "mv", output); + } else if c_host_src.exists() { + // Compile host.c, if it exists + let output = build_c_host_native( + &env_path, + &env_home, + host_dest_native.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); } } diff --git a/linker/Cargo.toml b/linker/Cargo.toml index 520ed8f660..b5babe9ebf 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -18,6 +18,7 @@ test = false bench = false [dependencies] +roc_mono = { path = "../compiler/mono" } roc_build = { path = "../compiler/build", default-features = false } roc_collections = { path = "../compiler/collections" } bumpalo = { version = "3.6", features = ["collections"] } @@ -29,3 +30,4 @@ object = { version = "0.26", features = ["read"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" target-lexicon = "0.12.2" +tempfile = "3.1.0" diff --git a/linker/src/lib.rs b/linker/src/lib.rs index c747d184f8..26eff91133 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -8,8 +8,9 @@ use object::{ Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, Symbol, SymbolIndex, SymbolSection, }; -use roc_build::link::LinkType; +use roc_build::link::{rebuild_host, LinkType}; use roc_collections::all::MutMap; +use roc_mono::ir::OptLevel; use std::cmp::Ordering; use std::convert::TryFrom; use std::ffi::CStr; @@ -21,6 +22,7 @@ use std::os::raw::c_char; use std::path::Path; use std::time::{Duration, SystemTime}; use target_lexicon::Triple; +use tempfile::{Builder, NamedTempFile}; mod metadata; @@ -132,27 +134,47 @@ pub fn supported(link_type: &LinkType, target: &Triple) -> bool { } pub fn build_and_preprocess_host( + opt_level: OptLevel, target: &Triple, host_input_path: &Path, exposed_to_host: Vec, ) -> io::Result<()> { let lib = generate_dynamic_lib(exposed_to_host)?; + rebuild_host(opt_level, target, host_input_path, Some(&lib.path())); Ok(()) } -fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result<()> { - Ok(()) +fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result { + for sym in exposed_to_host { + println!("{}", sym); + } + let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?; + Ok(dummy_lib_file) } +pub fn preprocess(matches: &ArgMatches) -> io::Result { + preprocess_impl( + &matches.value_of(EXEC).unwrap(), + &matches.value_of(METADATA).unwrap(), + &matches.value_of(OUT).unwrap(), + &matches.value_of(SHARED_LIB).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} // TODO: Most of this file is a mess of giant functions just to check if things work. // Clean it all up and refactor nicely. -pub fn preprocess(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); - +fn preprocess_impl( + exec_filename: &str, + metadata_filename: &str, + out_filename: &str, + shared_lib_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let exec_parsing_start = total_start; - let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?; + let exec_file = fs::File::open(exec_filename)?; let exec_mmap = unsafe { Mmap::map(&exec_file)? }; let exec_data = &*exec_mmap; let exec_obj = match object::File::parse(exec_data) { @@ -489,7 +511,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } }; - let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap()) + let shared_lib_name = Path::new(shared_lib_filename) .file_name() .unwrap() .to_str() @@ -623,7 +645,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { .write(true) .create(true) .truncate(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; out_file.set_len(md.exec_len)?; let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; @@ -884,7 +906,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } let saving_metadata_start = SystemTime::now(); - let output = fs::File::create(&matches.value_of(METADATA).unwrap())?; + let output = fs::File::create(metadata_filename)?; let output = BufWriter::new(output); if let Err(err) = serialize_into(output, &md) { println!("Failed to serialize metadata: {}", err); @@ -929,12 +951,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } pub fn surgery(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); + surgery_impl( + &matches.value_of(APP).unwrap(), + &matches.value_of(METADATA).unwrap(), + &matches.value_of(OUT).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} +fn surgery_impl( + app_filename: &str, + metadata_filename: &str, + out_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let loading_metadata_start = total_start; - let input = fs::File::open(&matches.value_of(METADATA).unwrap())?; + let input = fs::File::open(metadata_filename)?; let input = BufReader::new(input); let md: metadata::Metadata = match deserialize_from(input) { Ok(data) => data, @@ -946,7 +981,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); let app_parsing_start = SystemTime::now(); - let app_file = fs::File::open(&matches.value_of(APP).unwrap())?; + let app_file = fs::File::open(app_filename)?; let app_mmap = unsafe { Mmap::map(&app_file)? }; let app_data = &*app_mmap; let app_obj = match object::File::parse(app_data) { @@ -962,7 +997,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let exec_file = fs::OpenOptions::new() .read(true) .write(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; exec_file.set_len(max_out_len)?; From e2411ea83fce9f918b3b9bcb9b5b7b3bf0810607 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 14 Sep 2021 23:06:22 -0700 Subject: [PATCH 04/16] Add surgical linking to frontend with simple dummy lib creation --- Cargo.lock | 2 + cli/src/build.rs | 50 +++++++++++---------- examples/.gitignore | 3 ++ linker/Cargo.toml | 2 +- linker/src/lib.rs | 105 ++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 129 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c17cb03261..8d5d3bb0b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2517,7 +2517,9 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" dependencies = [ + "crc32fast", "flate2", + "indexmap", "memchr", ] diff --git a/cli/src/build.rs b/cli/src/build.rs index fec21f420b..f3db76b80a 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -232,21 +232,35 @@ pub fn build_file<'a>( // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); - let (mut child, binary_path) = // TODO use lld - link( - target, - binary_path, - &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], - link_type - ) - .map_err(|_| { - todo!("gracefully handle `rustc` failing to spawn."); + let outcome = if surgically_link { + roc_linker::link_preprocessed_host(target, &host_input_path, &app_o_file, &binary_path) + .map_err(|_| { + todo!("gracefully handle failing to surgically link"); + })?; + BuildOutcome::NoProblems + } else { + let (mut child, _) = // TODO use lld + link( + target, + binary_path.clone(), + &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], + link_type + ) + .map_err(|_| { + todo!("gracefully handle `ld` failing to spawn."); + })?; + + let exit_status = child.wait().map_err(|_| { + todo!("gracefully handle error after `ld` spawned"); })?; - let cmd_result = child.wait().map_err(|_| { - todo!("gracefully handle error after `rustc` spawned"); - }); - + // TODO change this to report whether there were errors or warnings! + if exit_status.success() { + BuildOutcome::NoProblems + } else { + BuildOutcome::Errors + } + }; let linking_time = link_start.elapsed().unwrap(); if emit_timings { @@ -255,16 +269,6 @@ pub fn build_file<'a>( let total_time = compilation_start.elapsed().unwrap(); - // If the cmd errored out, return the Err. - let exit_status = cmd_result?; - - // TODO change this to report whether there were errors or warnings! - let outcome = if exit_status.success() { - BuildOutcome::NoProblems - } else { - BuildOutcome::Errors - }; - Ok(BuiltFile { binary_path, outcome, diff --git a/examples/.gitignore b/examples/.gitignore index 874bbd86e5..000f652572 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,3 +4,6 @@ app libhost.a roc_app.ll roc_app.bc +dynhost +preprocessedhost +metadata diff --git a/linker/Cargo.toml b/linker/Cargo.toml index b5babe9ebf..843bf6caac 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -26,7 +26,7 @@ bumpalo = { version = "3.6", features = ["collections"] } clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } iced-x86 = "1.14" memmap2 = "0.3" -object = { version = "0.26", features = ["read"] } +object = { version = "0.26", features = ["read", "write"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" target-lexicon = "0.12.2" diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 26eff91133..fd26990c18 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -2,11 +2,12 @@ use bincode::{deserialize_from, serialize_into}; use clap::{App, AppSettings, Arg, ArgMatches}; use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind}; use memmap2::{Mmap, MmapMut}; +use object::write; use object::{elf, endian}; use object::{ - Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, LittleEndian, NativeEndian, - Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, - Symbol, SymbolIndex, SymbolSection, + Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian, + NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, + SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection, }; use roc_build::link::{rebuild_host, LinkType}; use roc_collections::all::MutMap; @@ -20,6 +21,7 @@ use std::io::{BufReader, BufWriter}; use std::mem; use std::os::raw::c_char; use std::path::Path; +use std::process::Command; use std::time::{Duration, SystemTime}; use target_lexicon::Triple; use tempfile::{Builder, NamedTempFile}; @@ -139,16 +141,100 @@ pub fn build_and_preprocess_host( host_input_path: &Path, exposed_to_host: Vec, ) -> io::Result<()> { - let lib = generate_dynamic_lib(exposed_to_host)?; - rebuild_host(opt_level, target, host_input_path, Some(&lib.path())); + let dummy_lib = generate_dynamic_lib(target, exposed_to_host)?; + let dummy_lib = dummy_lib.path(); + rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib)); + let dynhost = host_input_path.with_file_name("dynhost"); + let metadata = host_input_path.with_file_name("metadata"); + let prehost = host_input_path.with_file_name("preprocessedhost"); + if preprocess_impl( + dynhost.to_str().unwrap(), + metadata.to_str().unwrap(), + prehost.to_str().unwrap(), + dummy_lib.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to preprocess host"); + } Ok(()) } -fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result { - for sym in exposed_to_host { - println!("{}", sym); +pub fn link_preprocessed_host( + _target: &Triple, + host_input_path: &Path, + roc_app_obj: &Path, + binary_path: &Path, +) -> io::Result<()> { + let metadata = host_input_path.with_file_name("metadata"); + let prehost = host_input_path.with_file_name("preprocessedhost"); + if surgery_impl( + roc_app_obj.to_str().unwrap(), + metadata.to_str().unwrap(), + prehost.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to surgically link host"); } + std::fs::rename(prehost, binary_path) +} + +fn generate_dynamic_lib( + _target: &Triple, + exposed_to_host: Vec, +) -> io::Result { + let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?; + let dummy_obj_file = dummy_obj_file.path(); let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?; + + // TODO deal with other architectures here. + let mut out_object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let text_section = out_object.section_id(write::StandardSection::Text); + for sym in exposed_to_host { + out_object.add_symbol(write::Symbol { + name: format!("roc__{}_1_exposed", sym).as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } + std::fs::write( + &dummy_obj_file, + out_object.write().expect("failed to build output object"), + ) + .expect("failed to write object to file"); + + let output = Command::new("ld") + .args(&[ + "-shared", + dummy_obj_file.to_str().unwrap(), + "-o", + dummy_lib_file.path().to_str().unwrap(), + ]) + .output() + .unwrap(); + + if !output.status.success() { + match std::str::from_utf8(&output.stderr) { + Ok(stderr) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was:\n{}", + stderr + ), + Err(utf8_err) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})", + utf8_err + ), + } + } Ok(dummy_lib_file) } @@ -454,6 +540,7 @@ fn preprocess_impl( || inst.is_jmp_far_indirect() || inst.is_jmp_near_indirect()) && !indirect_warning_given + && verbose { indirect_warning_given = true; println!(); @@ -538,7 +625,7 @@ fn preprocess_impl( ) as usize; let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8; let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap(); - if c_str == shared_lib_name { + if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name { shared_lib_index = Some(dyn_lib_index); if verbose { println!( From e8e7f9cad8408e3b24a8674ac14d926cb0aba91f Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Wed, 15 Sep 2021 08:58:23 -0700 Subject: [PATCH 05/16] Add executable file permissions --- linker/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/linker/src/lib.rs b/linker/src/lib.rs index fd26990c18..7d57238ec9 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -20,6 +20,7 @@ use std::io; use std::io::{BufReader, BufWriter}; use std::mem; use std::os::raw::c_char; +use std::os::unix::fs::PermissionsExt; use std::path::Path; use std::process::Command; use std::time::{Duration, SystemTime}; @@ -1525,6 +1526,12 @@ fn surgery_impl( let flushing_data_duration = flushing_data_start.elapsed().unwrap(); exec_file.set_len(offset as u64 + 1)?; + + // Make sure the final executable has permision to execute. + let mut perms = fs::metadata(out_filename)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(out_filename, perms)?; + let total_duration = total_start.elapsed().unwrap(); if verbose || time { From da28b669bbea5f9a272c63284f8c71739d4ac1b9 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Wed, 15 Sep 2021 11:45:44 -0700 Subject: [PATCH 06/16] Get zig host working --- compiler/build/src/link.rs | 15 ++- examples/benchmarks/platform/host.zig | 10 ++ examples/effect/thing/platform-dir/host.zig | 10 ++ examples/hello-rust/platform/host.c | 9 +- examples/hello-world/platform/host.c | 120 ++++++++++---------- examples/hello-zig/platform/host.zig | 10 ++ examples/quicksort/platform/host.zig | 10 ++ linker/src/lib.rs | 57 ++++++---- linker/tests/fib/.gitignore | 6 +- 9 files changed, 157 insertions(+), 90 deletions(-) diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index d6de4b12d9..42f8d40bf9 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -92,9 +92,9 @@ pub fn build_zig_host_native( .env("PATH", env_path) .env("HOME", env_home); if let Some(shared_lib_path) = shared_lib_path { - command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); } else { - command.arg("build-obj"); + command.args(&["build-obj", "-fPIC"]); } command.args(&[ zig_host_src, @@ -108,7 +108,6 @@ pub fn build_zig_host_native( // include libc "--library", "c", - "-fPIC", // cross-compile? "-target", target, @@ -176,9 +175,9 @@ pub fn build_zig_host_native( .env("PATH", &env_path) .env("HOME", &env_home); if let Some(shared_lib_path) = shared_lib_path { - command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); } else { - command.arg("build-obj"); + command.args(&["build-obj", "-fPIC"]); } command.args(&[ zig_host_src, @@ -195,7 +194,6 @@ pub fn build_zig_host_native( // include libc "--library", "c", - "-fPIC", ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); @@ -267,10 +265,11 @@ pub fn build_c_host_native( .env("PATH", &env_path) .env("HOME", &env_home) .args(sources) - .args(&["-fPIC", "-o", dest]); + .args(&["-o", dest]); if let Some(shared_lib_path) = shared_lib_path { command.args(&[ shared_lib_path.to_str().unwrap(), + "-fPIE", "-lm", "-lpthread", "-ldl", @@ -278,7 +277,7 @@ pub fn build_c_host_native( "-lutil", ]); } else { - command.arg("-c"); + command.args(&["-fPIC", "-c"]); } if matches!(opt_level, OptLevel::Optimize) { command.arg("-O2"); diff --git a/examples/benchmarks/platform/host.zig b/examples/benchmarks/platform/host.zig index 503a66ed2d..36c84a8321 100644 --- a/examples/benchmarks/platform/host.zig +++ b/examples/benchmarks/platform/host.zig @@ -33,6 +33,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -74,6 +76,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub fn main() u8 { diff --git a/examples/effect/thing/platform-dir/host.zig b/examples/effect/thing/platform-dir/host.zig index 063d28973f..e9d02cb3a2 100644 --- a/examples/effect/thing/platform-dir/host.zig +++ b/examples/effect/thing/platform-dir/host.zig @@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { return malloc(size); @@ -52,6 +54,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub export fn main() u8 { diff --git a/examples/hello-rust/platform/host.c b/examples/hello-rust/platform/host.c index 0378c69589..9b91965724 100644 --- a/examples/hello-rust/platform/host.c +++ b/examples/hello-rust/platform/host.c @@ -1,7 +1,12 @@ #include +#include extern int rust_main(); -int main() { - return rust_main(); +int main() { return rust_main(); } + +void *roc_memcpy(void *dest, const void *src, size_t n) { + return memcpy(dest, src, n); } + +void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); } \ No newline at end of file diff --git a/examples/hello-world/platform/host.c b/examples/hello-world/platform/host.c index f968d0c763..4b45c20482 100644 --- a/examples/hello-world/platform/host.c +++ b/examples/hello-world/platform/host.c @@ -1,89 +1,91 @@ -#include -#include -#include #include -#include #include +#include +#include +#include +#include -void* roc_alloc(size_t size, unsigned int alignment) { - return malloc(size); +void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); } + +void* roc_realloc(void* ptr, size_t old_size, size_t new_size, + unsigned int alignment) { + return realloc(ptr, new_size); } -void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) { - return realloc(ptr, new_size); -} - -void roc_dealloc(void* ptr, unsigned int alignment) { - free(ptr); -} +void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); } void roc_panic(void* ptr, unsigned int alignment) { - char* msg = (char *)ptr; - fprintf(stderr, "Application crashed with message\n\n %s\n\nShutting down\n", msg); - exit(0); + char* msg = (char*)ptr; + fprintf(stderr, + "Application crashed with message\n\n %s\n\nShutting down\n", msg); + exit(0); } +void* roc_memcpy(void* dest, const void* src, size_t n) { + return memcpy(dest, src, n); +} + +void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); } + struct RocStr { - char* bytes; - size_t len; + char* bytes; + size_t len; }; -bool is_small_str(struct RocStr str) { - return ((ssize_t)str.len) < 0; -} +bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; } // Determine the length of the string, taking into // account the small string optimization size_t roc_str_len(struct RocStr str) { - char* bytes = (char*)&str; - char last_byte = bytes[sizeof(str) - 1]; - char last_byte_xored = last_byte ^ 0b10000000; - size_t small_len = (size_t)(last_byte_xored); - size_t big_len = str.len; + char* bytes = (char*)&str; + char last_byte = bytes[sizeof(str) - 1]; + char last_byte_xored = last_byte ^ 0b10000000; + size_t small_len = (size_t)(last_byte_xored); + size_t big_len = str.len; - // Avoid branch misprediction costs by always - // determining both small_len and big_len, - // so this compiles to a cmov instruction. - if (is_small_str(str)) { - return small_len; - } else { - return big_len; - } + // Avoid branch misprediction costs by always + // determining both small_len and big_len, + // so this compiles to a cmov instruction. + if (is_small_str(str)) { + return small_len; + } else { + return big_len; + } } struct RocCallResult { - size_t flag; - struct RocStr content; + size_t flag; + struct RocStr content; }; -extern void roc__mainForHost_1_exposed(struct RocCallResult *re); +extern void roc__mainForHost_1_exposed(struct RocCallResult* re); int main() { - // Make space for the Roc call result - struct RocCallResult call_result; + // Make space for the Roc call result + struct RocCallResult call_result; - // Call Roc to populate call_result - roc__mainForHost_1_exposed(&call_result); + // Call Roc to populate call_result + roc__mainForHost_1_exposed(&call_result); - // Determine str_len and the str_bytes pointer, - // taking into account the small string optimization. - struct RocStr str = call_result.content; - size_t str_len = roc_str_len(str); - char* str_bytes; + // Determine str_len and the str_bytes pointer, + // taking into account the small string optimization. + struct RocStr str = call_result.content; + size_t str_len = roc_str_len(str); + char* str_bytes; - if (is_small_str(str)) { - str_bytes = (char*)&str; - } else { - str_bytes = str.bytes; - } + if (is_small_str(str)) { + str_bytes = (char*)&str; + } else { + str_bytes = str.bytes; + } - // Write to stdout - if (write(1, str_bytes, str_len) >= 0) { - // Writing succeeded! - return 0; - } else { - printf("Error writing to stdout: %s\n", strerror(errno)); + // Write to stdout + if (write(1, str_bytes, str_len) >= 0) { + // Writing succeeded! + return 0; + } else { + printf("Error writing to stdout: %s\n", strerror(errno)); - return 1; - } + return 1; + } } diff --git a/examples/hello-zig/platform/host.zig b/examples/hello-zig/platform/host.zig index 8384c996d3..23e87920fb 100644 --- a/examples/hello-zig/platform/host.zig +++ b/examples/hello-zig/platform/host.zig @@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { _ = alignment; @@ -51,6 +53,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const mem = std.mem; const Allocator = mem.Allocator; diff --git a/examples/quicksort/platform/host.zig b/examples/quicksort/platform/host.zig index 38fb29f699..328562f28b 100644 --- a/examples/quicksort/platform/host.zig +++ b/examples/quicksort/platform/host.zig @@ -26,6 +26,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -67,6 +69,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + // warning! the array is currently stack-allocated so don't make this too big const NUM_NUMS = 100; diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 7d57238ec9..f17e1f4888 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -170,17 +170,18 @@ pub fn link_preprocessed_host( ) -> io::Result<()> { let metadata = host_input_path.with_file_name("metadata"); let prehost = host_input_path.with_file_name("preprocessedhost"); + std::fs::copy(prehost, binary_path)?; if surgery_impl( roc_app_obj.to_str().unwrap(), metadata.to_str().unwrap(), - prehost.to_str().unwrap(), + binary_path.to_str().unwrap(), false, false, )? != 0 { panic!("Failed to surgically link host"); } - std::fs::rename(prehost, binary_path) + Ok(()) } fn generate_dynamic_lib( @@ -197,16 +198,24 @@ fn generate_dynamic_lib( let text_section = out_object.section_id(write::StandardSection::Text); for sym in exposed_to_host { - out_object.add_symbol(write::Symbol { - name: format!("roc__{}_1_exposed", sym).as_bytes().to_vec(), - value: 0, - size: 0, - kind: SymbolKind::Text, - scope: SymbolScope::Dynamic, - weak: false, - section: write::SymbolSection::Section(text_section), - flags: SymbolFlags::None, - }); + for name in &[ + format!("roc__{}_1_exposed", sym), + format!("roc__{}_1_Fx_caller", sym), + format!("roc__{}_1_Fx_size", sym), + format!("roc__{}_1_Fx_result_size", sym), + format!("roc__{}_size", sym), + ] { + out_object.add_symbol(write::Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } } std::fs::write( &dummy_obj_file, @@ -994,16 +1003,22 @@ fn preprocess_impl( } let saving_metadata_start = SystemTime::now(); - let output = fs::File::create(metadata_filename)?; - let output = BufWriter::new(output); - if let Err(err) = serialize_into(output, &md) { - println!("Failed to serialize metadata: {}", err); - return Ok(-1); - }; + // This block ensure that the metadata is fully written and timed before continuing. + { + let output = fs::File::create(metadata_filename)?; + let output = BufWriter::new(output); + if let Err(err) = serialize_into(output, &md) { + println!("Failed to serialize metadata: {}", err); + return Ok(-1); + }; + } let saving_metadata_duration = saving_metadata_start.elapsed().unwrap(); let flushing_data_start = SystemTime::now(); out_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(out_mmap); + drop(out_file); let flushing_data_duration = flushing_data_start.elapsed().unwrap(); let total_duration = total_start.elapsed().unwrap(); @@ -1523,9 +1538,11 @@ fn surgery_impl( let flushing_data_start = SystemTime::now(); exec_mmap.flush()?; - let flushing_data_duration = flushing_data_start.elapsed().unwrap(); - + // Also drop files to to ensure data is fully written here. + drop(exec_mmap); exec_file.set_len(offset as u64 + 1)?; + drop(exec_file); + let flushing_data_duration = flushing_data_start.elapsed().unwrap(); // Make sure the final executable has permision to execute. let mut perms = fs::metadata(out_filename)?.permissions(); diff --git a/linker/tests/fib/.gitignore b/linker/tests/fib/.gitignore index a229b0fd25..3db2165e59 100644 --- a/linker/tests/fib/.gitignore +++ b/linker/tests/fib/.gitignore @@ -3,4 +3,8 @@ fib zig-cache zig-out -*.o \ No newline at end of file +*.o + +dynhost +preprocessedhost +metadata From b1e02315d003840f2a5091ea694af014b9495caf Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Wed, 15 Sep 2021 12:28:19 -0700 Subject: [PATCH 07/16] Strip debug info from zig --- compiler/build/src/link.rs | 3 +++ linker/src/lib.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 42f8d40bf9..1d5a834c55 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -108,6 +108,7 @@ pub fn build_zig_host_native( // include libc "--library", "c", + "--strip", // cross-compile? "-target", target, @@ -194,6 +195,7 @@ pub fn build_zig_host_native( // include libc "--library", "c", + "--strip", ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); @@ -244,6 +246,7 @@ pub fn build_zig_host_wasm32( // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", "-fPIC", + "--strip", ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); diff --git a/linker/src/lib.rs b/linker/src/lib.rs index f17e1f4888..3b1e71be90 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -198,6 +198,7 @@ fn generate_dynamic_lib( let text_section = out_object.section_id(write::StandardSection::Text); for sym in exposed_to_host { + // TODO properly generate this list. for name in &[ format!("roc__{}_1_exposed", sym), format!("roc__{}_1_Fx_caller", sym), From e4b3402369965cc6e9ceac828c8884be37342bd7 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Wed, 15 Sep 2021 15:16:39 -0700 Subject: [PATCH 08/16] Create dummy lib as libapp.so --- examples/.gitignore | 1 + linker/src/lib.rs | 16 +++++++++------- linker/tests/fib/.gitignore | 1 + 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/examples/.gitignore b/examples/.gitignore index 000f652572..c0e522cc76 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -7,3 +7,4 @@ roc_app.bc dynhost preprocessedhost metadata +libapp.so \ No newline at end of file diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 3b1e71be90..4d815c10b7 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -25,7 +25,7 @@ use std::path::Path; use std::process::Command; use std::time::{Duration, SystemTime}; use target_lexicon::Triple; -use tempfile::{Builder, NamedTempFile}; +use tempfile::Builder; mod metadata; @@ -142,8 +142,8 @@ pub fn build_and_preprocess_host( host_input_path: &Path, exposed_to_host: Vec, ) -> io::Result<()> { - let dummy_lib = generate_dynamic_lib(target, exposed_to_host)?; - let dummy_lib = dummy_lib.path(); + let dummy_lib = host_input_path.with_file_name("libapp.so"); + generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?; rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib)); let dynhost = host_input_path.with_file_name("dynhost"); let metadata = host_input_path.with_file_name("metadata"); @@ -187,10 +187,10 @@ pub fn link_preprocessed_host( fn generate_dynamic_lib( _target: &Triple, exposed_to_host: Vec, -) -> io::Result { + dummy_lib_path: &Path, +) -> io::Result<()> { let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?; let dummy_obj_file = dummy_obj_file.path(); - let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?; // TODO deal with other architectures here. let mut out_object = @@ -227,9 +227,11 @@ fn generate_dynamic_lib( let output = Command::new("ld") .args(&[ "-shared", + "-soname", + dummy_lib_path.file_name().unwrap().to_str().unwrap(), dummy_obj_file.to_str().unwrap(), "-o", - dummy_lib_file.path().to_str().unwrap(), + dummy_lib_path.to_str().unwrap(), ]) .output() .unwrap(); @@ -246,7 +248,7 @@ fn generate_dynamic_lib( ), } } - Ok(dummy_lib_file) + Ok(()) } pub fn preprocess(matches: &ArgMatches) -> io::Result { diff --git a/linker/tests/fib/.gitignore b/linker/tests/fib/.gitignore index 3db2165e59..a3c0d77f6d 100644 --- a/linker/tests/fib/.gitignore +++ b/linker/tests/fib/.gitignore @@ -8,3 +8,4 @@ zig-out dynhost preprocessedhost metadata +libapp.so \ No newline at end of file From b56606c5acec7e5f178ea46f1291a508f9b76918 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Wed, 15 Sep 2021 15:38:40 -0700 Subject: [PATCH 09/16] Add comment explaining rust host failure --- linker/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 4d815c10b7..3613489e4f 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -369,6 +369,9 @@ fn preprocess_impl( println!("PLT File Offset: {:+x}", plt_offset); } + // TODO: it looks like we may need to support global data host relocations. + // Rust host look to be using them by default instead of the plt. + // I think this is due to first linking into a static lib and then linking to the c wrapper. let plt_relocs = (match exec_obj.dynamic_relocations() { Some(relocs) => relocs, None => { From d8d147375df86735e99cc1938f33881527dd0fe8 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Thu, 16 Sep 2021 20:19:28 -0700 Subject: [PATCH 10/16] update fib gitignore --- examples/fib/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/fib/.gitignore b/examples/fib/.gitignore index 76d4bb83f8..41d1223f94 100644 --- a/examples/fib/.gitignore +++ b/examples/fib/.gitignore @@ -1 +1,2 @@ add +fib From 66a7a3aa074075122e9842f051305bde5a94fe8d Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Thu, 16 Sep 2021 22:34:55 -0700 Subject: [PATCH 11/16] Make clippy happy again --- cli/src/build.rs | 6 +++--- compiler/build/src/link.rs | 2 ++ linker/src/lib.rs | 14 +++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cli/src/build.rs b/cli/src/build.rs index eed7578b3e..defb8e0781 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -101,7 +101,7 @@ pub fn build_file<'a>( opt_level, surgically_link, host_input_path.clone(), - target.clone(), + target, loaded .exposed_to_host .keys() @@ -227,7 +227,7 @@ pub fn build_file<'a>( // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); let outcome = if surgically_link { - roc_linker::link_preprocessed_host(target, &host_input_path, &app_o_file, &binary_path) + roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path) .map_err(|_| { todo!("gracefully handle failing to surgically link"); })?; @@ -274,7 +274,7 @@ fn spawn_rebuild_thread( opt_level: OptLevel, surgically_link: bool, host_input_path: PathBuf, - target: Triple, + target: &Triple, exported_symbols: Vec, ) -> std::thread::JoinHandle { let thread_local_target = target.clone(); diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 1d5a834c55..6d61145cc1 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -76,6 +76,7 @@ fn find_wasi_libc_path() -> PathBuf { } #[cfg(not(target_os = "macos"))] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -120,6 +121,7 @@ pub fn build_zig_host_native( } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 3613489e4f..2324671cc9 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -253,10 +253,10 @@ fn generate_dynamic_lib( pub fn preprocess(matches: &ArgMatches) -> io::Result { preprocess_impl( - &matches.value_of(EXEC).unwrap(), - &matches.value_of(METADATA).unwrap(), - &matches.value_of(OUT).unwrap(), - &matches.value_of(SHARED_LIB).unwrap(), + matches.value_of(EXEC).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.value_of(SHARED_LIB).unwrap(), matches.is_present(FLAG_VERBOSE), matches.is_present(FLAG_TIME), ) @@ -1061,9 +1061,9 @@ fn preprocess_impl( pub fn surgery(matches: &ArgMatches) -> io::Result { surgery_impl( - &matches.value_of(APP).unwrap(), - &matches.value_of(METADATA).unwrap(), - &matches.value_of(OUT).unwrap(), + matches.value_of(APP).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), matches.is_present(FLAG_VERBOSE), matches.is_present(FLAG_TIME), ) From 4a6e6207059927210a933bffcbfd3a41ba8abfd6 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Sat, 18 Sep 2021 16:11:51 -0700 Subject: [PATCH 12/16] Add --precompiled-host flag to enable skipping rebuild --- cli/src/build.rs | 37 +++++++++++++++++++++---------------- cli/src/lib.rs | 15 +++++++++++++++ 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/cli/src/build.rs b/cli/src/build.rs index defb8e0781..fd5173ca3d 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -54,6 +54,7 @@ pub fn build_file<'a>( emit_timings: bool, link_type: LinkType, surgically_link: bool, + precompiled: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -100,6 +101,7 @@ pub fn build_file<'a>( let rebuild_thread = spawn_rebuild_thread( opt_level, surgically_link, + precompiled, host_input_path.clone(), target, loaded @@ -217,7 +219,7 @@ pub fn build_file<'a>( } let rebuild_duration = rebuild_thread.join().unwrap(); - if emit_timings { + if emit_timings && !precompiled { println!( "Finished rebuilding and preprocessing the host in {} ms\n", rebuild_duration @@ -273,6 +275,7 @@ pub fn build_file<'a>( fn spawn_rebuild_thread( opt_level: OptLevel, surgically_link: bool, + precompiled: bool, host_input_path: PathBuf, target: &Triple, exported_symbols: Vec, @@ -280,21 +283,23 @@ fn spawn_rebuild_thread( let thread_local_target = target.clone(); std::thread::spawn(move || { let rebuild_host_start = SystemTime::now(); - if surgically_link { - roc_linker::build_and_preprocess_host( - opt_level, - &thread_local_target, - host_input_path.as_path(), - exported_symbols, - ) - .unwrap(); - } else { - rebuild_host( - opt_level, - &thread_local_target, - host_input_path.as_path(), - None, - ); + if !precompiled { + if surgically_link { + roc_linker::build_and_preprocess_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); + } } let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); rebuild_host_end.as_millis() diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 963ba2e4bc..1bebaaece4 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -34,6 +34,7 @@ pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; pub const FLAG_TIME: &str = "time"; pub const FLAG_LINK: &str = "roc-linker"; +pub const FLAG_PRECOMPILED: &str = "precompiled-host"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -95,6 +96,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Uses the roc linker instead of the system linker.") .required(false), ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -175,6 +182,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Uses the roc linker instead of the system linker.") .required(false), ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -260,6 +273,7 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { LinkType::Executable }; let surgically_link = matches.is_present(FLAG_LINK); + let precompiled = matches.is_present(FLAG_PRECOMPILED); if surgically_link && !roc_linker::supported(&link_type, &target) { panic!( "Link type, {:?}, with target, {}, not supported by roc linker", @@ -299,6 +313,7 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { emit_timings, link_type, surgically_link, + precompiled, ); match res_binary_path { From 874ef7a5908fabb1b59dd4216e42cd2d9b9033b0 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Sat, 18 Sep 2021 17:54:02 -0700 Subject: [PATCH 13/16] Move copying precompiled host to rebuild thread --- cli/src/build.rs | 13 ++++++++++--- linker/src/lib.rs | 2 -- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cli/src/build.rs b/cli/src/build.rs index fd5173ca3d..9f6f46e51c 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -88,8 +88,10 @@ pub fn build_file<'a>( let app_extension = if emit_wasm { "bc" } else { "o" }; let cwd = roc_file_path.parent().unwrap(); - let path_to_platform = loaded.platform_path.clone(); + let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows + let mut host_input_path = PathBuf::from(cwd); + let path_to_platform = loaded.platform_path.clone(); host_input_path.push(&*path_to_platform); host_input_path.push("host"); host_input_path.set_extension(host_extension); @@ -103,6 +105,7 @@ pub fn build_file<'a>( surgically_link, precompiled, host_input_path.clone(), + binary_path.clone(), target, loaded .exposed_to_host @@ -169,8 +172,6 @@ pub fn build_file<'a>( program::report_problems(&mut loaded); let loaded = loaded; - let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows - let code_gen_timing = match opt_level { OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm( arena, @@ -277,6 +278,7 @@ fn spawn_rebuild_thread( surgically_link: bool, precompiled: bool, host_input_path: PathBuf, + binary_path: PathBuf, target: &Triple, exported_symbols: Vec, ) -> std::thread::JoinHandle { @@ -301,6 +303,11 @@ fn spawn_rebuild_thread( ); } } + if surgically_link { + // Copy preprocessed host to executable location. + let prehost = host_input_path.with_file_name("preprocessedhost"); + std::fs::copy(prehost, binary_path.as_path()).unwrap(); + } let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); rebuild_host_end.as_millis() }) diff --git a/linker/src/lib.rs b/linker/src/lib.rs index 2324671cc9..1a043fbce1 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -169,8 +169,6 @@ pub fn link_preprocessed_host( binary_path: &Path, ) -> io::Result<()> { let metadata = host_input_path.with_file_name("metadata"); - let prehost = host_input_path.with_file_name("preprocessedhost"); - std::fs::copy(prehost, binary_path)?; if surgery_impl( roc_app_obj.to_str().unwrap(), metadata.to_str().unwrap(), From cda29d07061016f2129c4748c604e8ca0b015a08 Mon Sep 17 00:00:00 2001 From: Richard Feldman Date: Sat, 18 Sep 2021 23:40:36 -0400 Subject: [PATCH 14/16] memcpy and memset are byte-aligned --- examples/benchmarks/platform/host.zig | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/benchmarks/platform/host.zig b/examples/benchmarks/platform/host.zig index b560c0cf6d..d2a7de8aeb 100644 --- a/examples/benchmarks/platform/host.zig +++ b/examples/benchmarks/platform/host.zig @@ -34,8 +34,8 @@ const Align = 2 * @alignOf(usize); extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void; extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void; -extern fn memcpy(dst: [*]align(Align) u8, src: [*]align(Align) u8, size: usize) callconv(.C) void; -extern fn memset(dst: [*]align(Align) u8, value: i32, size: usize) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -77,12 +77,12 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } -export fn roc_memcpy(dst: *c_void, src: *c_void, size: usize) callconv(.C) void{ - return memcpy(@alignCast(Align, @ptrCast([*]u8, dst)), @alignCast(Align, @ptrCast([*]u8, src)), size); +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); } -export fn roc_memset(dst: *c_void, value: i32, size: usize) callconv(.C) void{ - return memset(@alignCast(Align, @ptrCast([*]u8, dst)), value, size); +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); } const Unit = extern struct {}; From 9c1b3ff86afb058770a0e7eff2d2566c31e84f20 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Tue, 21 Sep 2021 17:17:20 -0700 Subject: [PATCH 15/16] Update TODO list for linker with next steps --- linker/README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/linker/README.md b/linker/README.md index 8b46b825d0..24ee2a3f61 100644 --- a/linker/README.md +++ b/linker/README.md @@ -29,13 +29,17 @@ This linker is run in 2 phases: preprocessing and surigical linking. 1. Surgically update all call locations in the platform 1. Surgically update call information in the application (also dealing with other relocations for builtins) -## TODO for merging with compiler flow +## TODO (In a lightly prioritized order) -1. Add new compiler flag to hide this all behind. -1. Get compiler to generate dummy shared libraries with Roc exported symbols defined. -1. Modify host linking to generate dynamic executable that links against the dummy lib. -1. Call the preprocessor on the dynamic executable host. -1. Call the surgical linker on the emitted roc object file and the preprocessed host. -1. Enjoy! -1. Extract preprocessing generation to run earlier, maybe in parallel with the main compiler until we have full precompiled hosts. -1. Maybe add a roc command to generate the dummy lib to be used by platform authors. +- Run CLI tests and/or benchmarks with the Roc Linker. +- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig). +- Investigate why C is using absolute jumps to the main function from `_start`. This means our shifts break the executable. +- Add Macho support + - Honestly should be almost exactly the same code. + This means we likely need to do a lot of refactoring to minimize the duplicate code. + The fun of almost but not quite the same. +- Add PE support + - As a prereq, we need roc building on Windows (I'm not sure it does currently). + - Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add. +- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl` +- Add a feature to the compiler to make this linker optional. From a593713800352177ded2c56182cd81ec98d3f35c Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Thu, 23 Sep 2021 21:29:53 -0700 Subject: [PATCH 16/16] Fix surgical linking for C hosts with extra arg --- compiler/build/src/link.rs | 1 + linker/README.md | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 528cb208a5..40a8f84cbf 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -275,6 +275,7 @@ pub fn build_c_host_native( command.args(&[ shared_lib_path.to_str().unwrap(), "-fPIE", + "-pie", "-lm", "-lpthread", "-ldl", diff --git a/linker/README.md b/linker/README.md index 24ee2a3f61..0d1e2d77ee 100644 --- a/linker/README.md +++ b/linker/README.md @@ -33,7 +33,6 @@ This linker is run in 2 phases: preprocessing and surigical linking. - Run CLI tests and/or benchmarks with the Roc Linker. - Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig). -- Investigate why C is using absolute jumps to the main function from `_start`. This means our shifts break the executable. - Add Macho support - Honestly should be almost exactly the same code. This means we likely need to do a lot of refactoring to minimize the duplicate code.