diff --git a/Cargo.lock b/Cargo.lock index 0f4f06be73..e8289983d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2517,7 +2517,9 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" dependencies = [ + "crc32fast", "flate2", + "indexmap", "memchr", ] @@ -3485,6 +3487,7 @@ dependencies = [ "roc_editor", "roc_fmt", "roc_gen_llvm", + "roc_linker", "roc_load", "roc_module", "roc_mono", @@ -3727,8 +3730,12 @@ dependencies = [ "iced-x86", "memmap2 0.3.1", "object 0.26.2", + "roc_build", "roc_collections", + "roc_mono", "serde", + "target-lexicon", + "tempfile", ] [[package]] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0225a11916..97ecb81a46 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false } roc_fmt = { path = "../compiler/fmt" } roc_reporting = { path = "../compiler/reporting" } roc_editor = { path = "../editor", optional = true } +roc_linker = { path = "../linker" } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } const_format = "0.2" diff --git a/cli/src/build.rs b/cli/src/build.rs index 21f945539a..d0702ce3dd 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -53,6 +53,8 @@ pub fn build_file<'a>( emit_debug_info: bool, emit_timings: bool, link_type: LinkType, + surgically_link: bool, + precompiled: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -85,7 +87,39 @@ pub fn build_file<'a>( let host_extension = if emit_wasm { "zig" } else { "o" }; let app_extension = if emit_wasm { "bc" } else { "o" }; + let cwd = roc_file_path.parent().unwrap(); + let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows + + if emit_wasm { + binary_path.set_extension("wasm"); + } + + let mut host_input_path = PathBuf::from(cwd); let path_to_platform = loaded.platform_path.clone(); + host_input_path.push(&*path_to_platform); + host_input_path.push("host"); + host_input_path.set_extension(host_extension); + + // TODO this should probably be moved before load_and_monomorphize. + // To do this we will need to preprocess files just for their exported symbols. + // Also, we should no longer need to do this once we have platforms on + // a package repository, as we can then get precompiled hosts from there. + let rebuild_thread = spawn_rebuild_thread( + opt_level, + surgically_link, + precompiled, + host_input_path.clone(), + binary_path.clone(), + target, + loaded + .exposed_to_host + .keys() + .map(|x| x.as_str(&loaded.interns).to_string()) + .collect(), + ); + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. let app_o_file = Builder::new() .prefix("roc_app") .suffix(&format!(".{}", app_extension)) @@ -142,13 +176,6 @@ pub fn build_file<'a>( program::report_problems(&mut loaded); let loaded = loaded; - let cwd = roc_file_path.parent().unwrap(); - let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows - - if emit_wasm { - binary_path.set_extension("wasm"); - } - let code_gen_timing = match opt_level { OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm( arena, @@ -196,44 +223,45 @@ pub fn build_file<'a>( ); } - // Step 2: link the precompiled host and compiled app - let mut host_input_path = PathBuf::from(cwd); - - host_input_path.push(&*path_to_platform); - host_input_path.push("host"); - host_input_path.set_extension(host_extension); - - // TODO we should no longer need to do this once we have platforms on - // a package repository, as we can then get precompiled hosts from there. - let rebuild_host_start = SystemTime::now(); - rebuild_host(target, host_input_path.as_path()); - let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); - - if emit_timings { + let rebuild_duration = rebuild_thread.join().unwrap(); + if emit_timings && !precompiled { println!( - "Finished rebuilding the host in {} ms\n", - rebuild_host_end.as_millis() + "Finished rebuilding and preprocessing the host in {} ms\n", + rebuild_duration ); } - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. + // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); - let (mut child, binary_path) = // TODO use lld - link( - target, - binary_path, - &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], - link_type - ) - .map_err(|_| { - todo!("gracefully handle `rustc` failing to spawn."); + let outcome = if surgically_link { + roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path) + .map_err(|_| { + todo!("gracefully handle failing to surgically link"); + })?; + BuildOutcome::NoProblems + } else { + let (mut child, _) = // TODO use lld + link( + target, + binary_path.clone(), + &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], + link_type + ) + .map_err(|_| { + todo!("gracefully handle `ld` failing to spawn."); + })?; + + let exit_status = child.wait().map_err(|_| { + todo!("gracefully handle error after `ld` spawned"); })?; - let cmd_result = child.wait().map_err(|_| { - todo!("gracefully handle error after `rustc` spawned"); - }); - + // TODO change this to report whether there were errors or warnings! + if exit_status.success() { + BuildOutcome::NoProblems + } else { + BuildOutcome::Errors + } + }; let linking_time = link_start.elapsed().unwrap(); if emit_timings { @@ -242,16 +270,6 @@ pub fn build_file<'a>( let total_time = compilation_start.elapsed().unwrap(); - // If the cmd errored out, return the Err. - let exit_status = cmd_result?; - - // TODO change this to report whether there were errors or warnings! - let outcome = if exit_status.success() { - BuildOutcome::NoProblems - } else { - BuildOutcome::Errors - }; - Ok(BuiltFile { binary_path, outcome, @@ -259,6 +277,46 @@ pub fn build_file<'a>( }) } +fn spawn_rebuild_thread( + opt_level: OptLevel, + surgically_link: bool, + precompiled: bool, + host_input_path: PathBuf, + binary_path: PathBuf, + target: &Triple, + exported_symbols: Vec, +) -> std::thread::JoinHandle { + let thread_local_target = target.clone(); + std::thread::spawn(move || { + let rebuild_host_start = SystemTime::now(); + if !precompiled { + if surgically_link { + roc_linker::build_and_preprocess_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); + } + } + if surgically_link { + // Copy preprocessed host to executable location. + let prehost = host_input_path.with_file_name("preprocessedhost"); + std::fs::copy(prehost, binary_path.as_path()).unwrap(); + } + let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); + rebuild_host_end.as_millis() + }) +} + #[allow(clippy::too_many_arguments)] pub fn check_file( arena: &Bump, diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 618c152f3c..c7b14a56f8 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -34,6 +34,8 @@ pub const FLAG_OPTIMIZE: &str = "optimize"; pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; pub const FLAG_TIME: &str = "time"; +pub const FLAG_LINK: &str = "roc-linker"; +pub const FLAG_PRECOMPILED: &str = "precompiled-host"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -89,6 +91,18 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -177,6 +191,18 @@ pub fn build_app<'a>() -> App<'a> { .help("Prints detailed compilation time information.") .required(false), ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -261,6 +287,14 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { } else { LinkType::Executable }; + let surgically_link = matches.is_present(FLAG_LINK); + let precompiled = matches.is_present(FLAG_PRECOMPILED); + if surgically_link && !roc_linker::supported(&link_type, &target) { + panic!( + "Link type, {:?}, with target, {}, not supported by roc linker", + link_type, target + ); + } let path = Path::new(filename); @@ -293,6 +327,8 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { emit_debug_info, emit_timings, link_type, + surgically_link, + precompiled, ); match res_binary_path { diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 1103ca6e2f..40a8f84cbf 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -76,6 +76,7 @@ fn find_wasi_libc_path() -> PathBuf { } #[cfg(not(target_os = "macos"))] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -83,36 +84,44 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) - .env("HOME", env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "-fcompiler-rt", - // include libc - "--library", - "c", - "-fPIC", - "-O", - "ReleaseSafe", - // cross-compile? - "-target", - target, - ]) - .output() - .unwrap() + .env("HOME", env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "-fcompiler-rt", + // include libc + "--library", + "c", + "--strip", + // cross-compile? + "-target", + target, + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -120,6 +129,8 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, _target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { use serde_json::Value; @@ -161,32 +172,37 @@ pub fn build_zig_host_native( zig_compiler_rt_path.push("special"); zig_compiler_rt_path.push("compiler_rt.zig"); - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", &env_path) - .env("HOME", &env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "--pkg-begin", - "compiler_rt", - zig_compiler_rt_path.to_str().unwrap(), - "--pkg-end", - // include libc - "--library", - "c", - "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + .env("HOME", &env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "--pkg-begin", + "compiler_rt", + zig_compiler_rt_path.to_str().unwrap(), + "--pkg-end", + // include libc + "--library", + "c", + "--strip", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } pub fn build_zig_host_wasm32( @@ -195,7 +211,12 @@ pub fn build_zig_host_wasm32( emit_bin: &str, zig_host_src: &str, zig_str_path: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { + if shared_lib_path.is_some() { + unimplemented!("Linking a shared library to wasm not yet implemented"); + } // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used // @@ -204,7 +225,8 @@ pub fn build_zig_host_wasm32( // we'd like to compile with `-target wasm32-wasi` but that is blocked on // // https://github.com/ziglang/zig/issues/9414 - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -226,21 +248,66 @@ pub fn build_zig_host_wasm32( // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", "-fPIC", - "-O", - "ReleaseSafe", - ]) - .output() - .unwrap() + "--strip", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } -pub fn rebuild_host(target: &Triple, host_input_path: &Path) { +pub fn build_c_host_native( + env_path: &str, + env_home: &str, + dest: &str, + sources: &[&str], + opt_level: OptLevel, + shared_lib_path: Option<&Path>, +) -> Output { + let mut command = Command::new("clang"); + command + .env_clear() + .env("PATH", &env_path) + .env("HOME", &env_home) + .args(sources) + .args(&["-o", dest]); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + shared_lib_path.to_str().unwrap(), + "-fPIE", + "-pie", + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + ]); + } else { + command.args(&["-fPIC", "-c"]); + } + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + command.output().unwrap() +} + +pub fn rebuild_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + shared_lib_path: Option<&Path>, +) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); - let host_dest_native = host_input_path.with_file_name("host.o"); + let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() { + "dynhost" + } else { + "host.o" + }); let host_dest_wasm = host_input_path.with_file_name("host.bc"); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); @@ -266,6 +333,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { &emit_bin, zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), + opt_level, + shared_lib_path, ) } Architecture::X86_64 => { @@ -277,6 +346,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "native", + opt_level, + shared_lib_path, ) } Architecture::X86_32(_) => { @@ -288,89 +359,142 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "i386-linux-musl", + opt_level, + shared_lib_path, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), }; validate_output("host.zig", "zig", output) - } else { - // Compile host.c - let output = Command::new("clang") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-O2", - "-fPIC", - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); - - validate_output("host.c", "clang", output); - } - - if cargo_host_src.exists() { + } else if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); - let libhost_dir = cargo_dir.join("target").join("release"); + let libhost_dir = + cargo_dir + .join("target") + .join(if matches!(opt_level, OptLevel::Optimize) { + "release" + } else { + "debug" + }); + let libhost = libhost_dir.join("libhost.a"); - let output = Command::new("cargo") - .args(&["build", "--release"]) - .current_dir(cargo_dir) - .output() - .unwrap(); + let mut command = Command::new("cargo"); + command.arg("build").current_dir(cargo_dir); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("--release"); + } + let output = command.output().unwrap(); - validate_output("src/lib.rs", "cargo build --release", output); + validate_output("src/lib.rs", "cargo build", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - "-L", - libhost_dir.to_str().unwrap(), - c_host_dest.to_str().unwrap(), - "-lhost", - "-o", + // Cargo hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); - validate_output("c_host.o", "ld", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + "-L", + libhost_dir.to_str().unwrap(), + c_host_dest.to_str().unwrap(), + "-lhost", + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); + validate_output("c_host.o", "ld", output); + + // Clean up c_host.o + let output = Command::new("rm") + .env_clear() + .args(&["-f", c_host_dest.to_str().unwrap()]) + .output() + .unwrap(); + + validate_output("rust_host.o", "rm", output); + } } else if rust_host_src.exists() { // Compile and link host.rs, if it exists - let output = Command::new("rustc") - .args(&[ - rust_host_src.to_str().unwrap(), - "-o", - rust_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("rustc"); + command.args(&[ + rust_host_src.to_str().unwrap(), + "-o", + rust_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O"); + } + let output = command.output().unwrap(); validate_output("host.rs", "rustc", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - c_host_dest.to_str().unwrap(), - rust_host_dest.to_str().unwrap(), - "-o", + // Rust hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[ + c_host_src.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + ], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); - validate_output("rust_host.o", "ld", output); + validate_output("host.c", "clang", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + c_host_dest.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); - // Clean up rust_host.o + validate_output("rust_host.o", "ld", output); + } + + // Clean up rust_host.o and c_host.o let output = Command::new("rm") .env_clear() .args(&[ @@ -382,15 +506,17 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { .unwrap(); validate_output("rust_host.o", "rm", output); - } else if c_host_dest.exists() { - // Clean up c_host.o - let output = Command::new("mv") - .env_clear() - .args(&[c_host_dest, host_dest_native]) - .output() - .unwrap(); - - validate_output("c_host.o", "mv", output); + } else if c_host_src.exists() { + // Compile host.c, if it exists + let output = build_c_host_native( + &env_path, + &env_home, + host_dest_native.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); } } diff --git a/examples/.gitignore b/examples/.gitignore index 874bbd86e5..c0e522cc76 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,3 +4,7 @@ app libhost.a roc_app.ll roc_app.bc +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file diff --git a/examples/benchmarks/platform/host.zig b/examples/benchmarks/platform/host.zig index 91967299b4..aa339ec306 100644 --- a/examples/benchmarks/platform/host.zig +++ b/examples/benchmarks/platform/host.zig @@ -29,10 +29,13 @@ extern fn roc__mainForHost_1_Fx_caller(*const u8, [*]u8, [*]u8) void; extern fn roc__mainForHost_1_Fx_size() i64; extern fn roc__mainForHost_1_Fx_result_size() i64; + const Align = 2 * @alignOf(usize); extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void; extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -74,6 +77,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub export fn main() callconv(.C) u8 { diff --git a/examples/effect/thing/platform-dir/host.zig b/examples/effect/thing/platform-dir/host.zig index d62dae1936..49fad7a204 100644 --- a/examples/effect/thing/platform-dir/host.zig +++ b/examples/effect/thing/platform-dir/host.zig @@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { return malloc(size); @@ -52,6 +54,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub export fn main() u8 { diff --git a/examples/fib/.gitignore b/examples/fib/.gitignore index 8cd68df839..41d1223f94 100644 --- a/examples/fib/.gitignore +++ b/examples/fib/.gitignore @@ -1,2 +1,2 @@ add -fib \ No newline at end of file +fib diff --git a/examples/hello-rust/platform/host.c b/examples/hello-rust/platform/host.c index 0378c69589..9b91965724 100644 --- a/examples/hello-rust/platform/host.c +++ b/examples/hello-rust/platform/host.c @@ -1,7 +1,12 @@ #include +#include extern int rust_main(); -int main() { - return rust_main(); +int main() { return rust_main(); } + +void *roc_memcpy(void *dest, const void *src, size_t n) { + return memcpy(dest, src, n); } + +void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); } \ No newline at end of file diff --git a/examples/hello-world/platform/host.c b/examples/hello-world/platform/host.c index 04615c7b40..99ada123fe 100644 --- a/examples/hello-world/platform/host.c +++ b/examples/hello-world/platform/host.c @@ -1,81 +1,81 @@ -#include -#include -#include #include -#include #include +#include +#include +#include +#include -void* roc_alloc(size_t size, unsigned int alignment) { - return malloc(size); +void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); } + +void* roc_realloc(void* ptr, size_t old_size, size_t new_size, + unsigned int alignment) { + return realloc(ptr, new_size); } -void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) { - return realloc(ptr, new_size); -} - -void roc_dealloc(void* ptr, unsigned int alignment) { - free(ptr); -} +void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); } void roc_panic(void* ptr, unsigned int alignment) { - char* msg = (char *)ptr; - fprintf(stderr, "Application crashed with message\n\n %s\n\nShutting down\n", msg); - exit(0); + char* msg = (char*)ptr; + fprintf(stderr, + "Application crashed with message\n\n %s\n\nShutting down\n", msg); + exit(0); } +void* roc_memcpy(void* dest, const void* src, size_t n) { + return memcpy(dest, src, n); +} + +void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); } + struct RocStr { - char* bytes; - size_t len; + char* bytes; + size_t len; }; -bool is_small_str(struct RocStr str) { - return ((ssize_t)str.len) < 0; -} +bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; } // Determine the length of the string, taking into // account the small string optimization size_t roc_str_len(struct RocStr str) { - char* bytes = (char*)&str; - char last_byte = bytes[sizeof(str) - 1]; - char last_byte_xored = last_byte ^ 0b10000000; - size_t small_len = (size_t)(last_byte_xored); - size_t big_len = str.len; + char* bytes = (char*)&str; + char last_byte = bytes[sizeof(str) - 1]; + char last_byte_xored = last_byte ^ 0b10000000; + size_t small_len = (size_t)(last_byte_xored); + size_t big_len = str.len; - // Avoid branch misprediction costs by always - // determining both small_len and big_len, - // so this compiles to a cmov instruction. - if (is_small_str(str)) { - return small_len; - } else { - return big_len; - } + // Avoid branch misprediction costs by always + // determining both small_len and big_len, + // so this compiles to a cmov instruction. + if (is_small_str(str)) { + return small_len; + } else { + return big_len; + } } extern struct RocStr roc__mainForHost_1_exposed(); int main() { - // Call Roc to populate call_result - struct RocStr call_result = roc__mainForHost_1_exposed(); + struct RocStr str = roc__mainForHost_1_exposed(); - // Determine str_len and the str_bytes pointer, - // taking into account the small string optimization. - struct RocStr str = call_result; - size_t str_len = roc_str_len(str); - char* str_bytes; + // Determine str_len and the str_bytes pointer, + // taking into account the small string optimization. + size_t str_len = roc_str_len(str); + char* str_bytes; - if (is_small_str(str)) { - str_bytes = (char*)&str; - } else { - str_bytes = str.bytes; - } + if (is_small_str(str)) { + str_bytes = (char*)&str; + } else { + str_bytes = str.bytes; + } - // Write to stdout - if (write(1, str_bytes, str_len) >= 0) { - // Writing succeeded! - return 0; - } else { - printf("Error writing to stdout: %s\n", strerror(errno)); + // Write to stdout + if (write(1, str_bytes, str_len) >= 0) { + // Writing succeeded! + return 0; + } else { + printf("Error writing to stdout: %s\n", strerror(errno)); - return 1; - } + return 1; + } } diff --git a/examples/hello-zig/platform/host.zig b/examples/hello-zig/platform/host.zig index d1f3ddb287..3c10412eff 100644 --- a/examples/hello-zig/platform/host.zig +++ b/examples/hello-zig/platform/host.zig @@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { _ = alignment; @@ -51,6 +53,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const mem = std.mem; const Allocator = mem.Allocator; diff --git a/examples/quicksort/platform/host.zig b/examples/quicksort/platform/host.zig index 94c423429e..fe8e90e490 100644 --- a/examples/quicksort/platform/host.zig +++ b/examples/quicksort/platform/host.zig @@ -26,6 +26,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -67,6 +69,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + // warning! the array is currently stack-allocated so don't make this too big const NUM_NUMS = 100; diff --git a/linker/Cargo.toml b/linker/Cargo.toml index 43461fc276..843bf6caac 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -18,12 +18,16 @@ test = false bench = false [dependencies] +roc_mono = { path = "../compiler/mono" } +roc_build = { path = "../compiler/build", default-features = false } roc_collections = { path = "../compiler/collections" } bumpalo = { version = "3.6", features = ["collections"] } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } iced-x86 = "1.14" memmap2 = "0.3" -object = { version = "0.26", features = ["read"] } +object = { version = "0.26", features = ["read", "write"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" +target-lexicon = "0.12.2" +tempfile = "3.1.0" diff --git a/linker/README.md b/linker/README.md index 8b46b825d0..0d1e2d77ee 100644 --- a/linker/README.md +++ b/linker/README.md @@ -29,13 +29,16 @@ This linker is run in 2 phases: preprocessing and surigical linking. 1. Surgically update all call locations in the platform 1. Surgically update call information in the application (also dealing with other relocations for builtins) -## TODO for merging with compiler flow +## TODO (In a lightly prioritized order) -1. Add new compiler flag to hide this all behind. -1. Get compiler to generate dummy shared libraries with Roc exported symbols defined. -1. Modify host linking to generate dynamic executable that links against the dummy lib. -1. Call the preprocessor on the dynamic executable host. -1. Call the surgical linker on the emitted roc object file and the preprocessed host. -1. Enjoy! -1. Extract preprocessing generation to run earlier, maybe in parallel with the main compiler until we have full precompiled hosts. -1. Maybe add a roc command to generate the dummy lib to be used by platform authors. +- Run CLI tests and/or benchmarks with the Roc Linker. +- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig). +- Add Macho support + - Honestly should be almost exactly the same code. + This means we likely need to do a lot of refactoring to minimize the duplicate code. + The fun of almost but not quite the same. +- Add PE support + - As a prereq, we need roc building on Windows (I'm not sure it does currently). + - Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add. +- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl` +- Add a feature to the compiler to make this linker optional. diff --git a/linker/src/lib.rs b/linker/src/lib.rs index e5e6b6f1a8..1a043fbce1 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -2,13 +2,16 @@ use bincode::{deserialize_from, serialize_into}; use clap::{App, AppSettings, Arg, ArgMatches}; use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind}; use memmap2::{Mmap, MmapMut}; +use object::write; use object::{elf, endian}; use object::{ - Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, LittleEndian, NativeEndian, - Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, - Symbol, SymbolIndex, SymbolSection, + Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian, + NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, + SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection, }; +use roc_build::link::{rebuild_host, LinkType}; use roc_collections::all::MutMap; +use roc_mono::ir::OptLevel; use std::cmp::Ordering; use std::convert::TryFrom; use std::ffi::CStr; @@ -17,8 +20,12 @@ use std::io; use std::io::{BufReader, BufWriter}; use std::mem; use std::os::raw::c_char; +use std::os::unix::fs::PermissionsExt; use std::path::Path; +use std::process::Command; use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; +use tempfile::Builder; mod metadata; @@ -122,15 +129,149 @@ pub fn build_app<'a>() -> App<'a> { ) } +pub fn supported(link_type: &LinkType, target: &Triple) -> bool { + link_type == &LinkType::Executable + && target.architecture == target_lexicon::Architecture::X86_64 + && target.operating_system == target_lexicon::OperatingSystem::Linux + && target.binary_format == target_lexicon::BinaryFormat::Elf +} + +pub fn build_and_preprocess_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + exposed_to_host: Vec, +) -> io::Result<()> { + let dummy_lib = host_input_path.with_file_name("libapp.so"); + generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?; + rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib)); + let dynhost = host_input_path.with_file_name("dynhost"); + let metadata = host_input_path.with_file_name("metadata"); + let prehost = host_input_path.with_file_name("preprocessedhost"); + if preprocess_impl( + dynhost.to_str().unwrap(), + metadata.to_str().unwrap(), + prehost.to_str().unwrap(), + dummy_lib.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to preprocess host"); + } + Ok(()) +} + +pub fn link_preprocessed_host( + _target: &Triple, + host_input_path: &Path, + roc_app_obj: &Path, + binary_path: &Path, +) -> io::Result<()> { + let metadata = host_input_path.with_file_name("metadata"); + if surgery_impl( + roc_app_obj.to_str().unwrap(), + metadata.to_str().unwrap(), + binary_path.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to surgically link host"); + } + Ok(()) +} + +fn generate_dynamic_lib( + _target: &Triple, + exposed_to_host: Vec, + dummy_lib_path: &Path, +) -> io::Result<()> { + let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?; + let dummy_obj_file = dummy_obj_file.path(); + + // TODO deal with other architectures here. + let mut out_object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let text_section = out_object.section_id(write::StandardSection::Text); + for sym in exposed_to_host { + // TODO properly generate this list. + for name in &[ + format!("roc__{}_1_exposed", sym), + format!("roc__{}_1_Fx_caller", sym), + format!("roc__{}_1_Fx_size", sym), + format!("roc__{}_1_Fx_result_size", sym), + format!("roc__{}_size", sym), + ] { + out_object.add_symbol(write::Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } + } + std::fs::write( + &dummy_obj_file, + out_object.write().expect("failed to build output object"), + ) + .expect("failed to write object to file"); + + let output = Command::new("ld") + .args(&[ + "-shared", + "-soname", + dummy_lib_path.file_name().unwrap().to_str().unwrap(), + dummy_obj_file.to_str().unwrap(), + "-o", + dummy_lib_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + + if !output.status.success() { + match std::str::from_utf8(&output.stderr) { + Ok(stderr) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was:\n{}", + stderr + ), + Err(utf8_err) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})", + utf8_err + ), + } + } + Ok(()) +} + +pub fn preprocess(matches: &ArgMatches) -> io::Result { + preprocess_impl( + matches.value_of(EXEC).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.value_of(SHARED_LIB).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} // TODO: Most of this file is a mess of giant functions just to check if things work. // Clean it all up and refactor nicely. -pub fn preprocess(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); - +fn preprocess_impl( + exec_filename: &str, + metadata_filename: &str, + out_filename: &str, + shared_lib_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let exec_parsing_start = total_start; - let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?; + let exec_file = fs::File::open(exec_filename)?; let exec_mmap = unsafe { Mmap::map(&exec_file)? }; let exec_data = &*exec_mmap; let exec_obj = match object::File::parse(exec_data) { @@ -226,6 +367,9 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { println!("PLT File Offset: {:+x}", plt_offset); } + // TODO: it looks like we may need to support global data host relocations. + // Rust host look to be using them by default instead of the plt. + // I think this is due to first linking into a static lib and then linking to the c wrapper. let plt_relocs = (match exec_obj.dynamic_relocations() { Some(relocs) => relocs, None => { @@ -410,6 +554,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { || inst.is_jmp_far_indirect() || inst.is_jmp_near_indirect()) && !indirect_warning_given + && verbose { indirect_warning_given = true; println!(); @@ -467,7 +612,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } }; - let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap()) + let shared_lib_name = Path::new(shared_lib_filename) .file_name() .unwrap() .to_str() @@ -494,7 +639,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { ) as usize; let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8; let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap(); - if c_str == shared_lib_name { + if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name { shared_lib_index = Some(dyn_lib_index); if verbose { println!( @@ -601,7 +746,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { .write(true) .create(true) .truncate(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; out_file.set_len(md.exec_len)?; let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; @@ -862,16 +1007,22 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } let saving_metadata_start = SystemTime::now(); - let output = fs::File::create(&matches.value_of(METADATA).unwrap())?; - let output = BufWriter::new(output); - if let Err(err) = serialize_into(output, &md) { - println!("Failed to serialize metadata: {}", err); - return Ok(-1); - }; + // This block ensure that the metadata is fully written and timed before continuing. + { + let output = fs::File::create(metadata_filename)?; + let output = BufWriter::new(output); + if let Err(err) = serialize_into(output, &md) { + println!("Failed to serialize metadata: {}", err); + return Ok(-1); + }; + } let saving_metadata_duration = saving_metadata_start.elapsed().unwrap(); let flushing_data_start = SystemTime::now(); out_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(out_mmap); + drop(out_file); let flushing_data_duration = flushing_data_start.elapsed().unwrap(); let total_duration = total_start.elapsed().unwrap(); @@ -907,12 +1058,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } pub fn surgery(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); + surgery_impl( + matches.value_of(APP).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} +fn surgery_impl( + app_filename: &str, + metadata_filename: &str, + out_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let loading_metadata_start = total_start; - let input = fs::File::open(&matches.value_of(METADATA).unwrap())?; + let input = fs::File::open(metadata_filename)?; let input = BufReader::new(input); let md: metadata::Metadata = match deserialize_from(input) { Ok(data) => data, @@ -924,7 +1088,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); let app_parsing_start = SystemTime::now(); - let app_file = fs::File::open(&matches.value_of(APP).unwrap())?; + let app_file = fs::File::open(app_filename)?; let app_mmap = unsafe { Mmap::map(&app_file)? }; let app_data = &*app_mmap; let app_obj = match object::File::parse(app_data) { @@ -940,7 +1104,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let exec_file = fs::OpenOptions::new() .read(true) .write(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; exec_file.set_len(max_out_len)?; @@ -1378,9 +1542,17 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let flushing_data_start = SystemTime::now(); exec_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(exec_mmap); + exec_file.set_len(offset as u64 + 1)?; + drop(exec_file); let flushing_data_duration = flushing_data_start.elapsed().unwrap(); - exec_file.set_len(offset as u64 + 1)?; + // Make sure the final executable has permision to execute. + let mut perms = fs::metadata(out_filename)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(out_filename, perms)?; + let total_duration = total_start.elapsed().unwrap(); if verbose || time { diff --git a/linker/tests/fib/.gitignore b/linker/tests/fib/.gitignore index a229b0fd25..a3c0d77f6d 100644 --- a/linker/tests/fib/.gitignore +++ b/linker/tests/fib/.gitignore @@ -3,4 +3,9 @@ fib zig-cache zig-out -*.o \ No newline at end of file +*.o + +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file