Merge remote-tracking branch 'origin/trunk' into hello-web

Cargo.lock

@@ -2517,7 +2517,9 @@ version = "0.26.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2"
 dependencies = [
+ "crc32fast",
  "flate2",
+ "indexmap",
  "memchr",
 ]
 
@@ -3485,6 +3487,7 @@ dependencies = [
  "roc_editor",
  "roc_fmt",
  "roc_gen_llvm",
+ "roc_linker",
  "roc_load",
  "roc_module",
  "roc_mono",
@@ -3727,8 +3730,12 @@ dependencies = [
  "iced-x86",
  "memmap2 0.3.1",
  "object 0.26.2",
+ "roc_build",
  "roc_collections",
+ "roc_mono",
  "serde",
+ "target-lexicon",
+ "tempfile",
 ]
 
 [[package]]

cli/Cargo.toml

@@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false }
 roc_fmt = { path = "../compiler/fmt" }
 roc_reporting = { path = "../compiler/reporting" }
 roc_editor = { path = "../editor", optional = true }
+roc_linker = { path = "../linker" }
 # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
 clap = { git = "https://github.com/rtfeldman/clap", branch = "master" }
 const_format = "0.2"

cli/src/build.rs

@@ -53,6 +53,8 @@ pub fn build_file<'a>(
     emit_debug_info: bool,
     emit_timings: bool,
     link_type: LinkType,
+    surgically_link: bool,
+    precompiled: bool,
 ) -> Result<BuiltFile, LoadingProblem<'a>> {
     let compilation_start = SystemTime::now();
     let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
@@ -85,7 +87,39 @@ pub fn build_file<'a>(
     let host_extension = if emit_wasm { "zig" } else { "o" };
     let app_extension = if emit_wasm { "bc" } else { "o" };
 
+    let cwd = roc_file_path.parent().unwrap();
+    let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows
+
+    if emit_wasm {
+        binary_path.set_extension("wasm");
+    }
+
+    let mut host_input_path = PathBuf::from(cwd);
     let path_to_platform = loaded.platform_path.clone();
+    host_input_path.push(&*path_to_platform);
+    host_input_path.push("host");
+    host_input_path.set_extension(host_extension);
+
+    // TODO this should probably be moved before load_and_monomorphize.
+    // To do this we will need to preprocess files just for their exported symbols.
+    // Also, we should no longer need to do this once we have platforms on
+    // a package repository, as we can then get precompiled hosts from there.
+    let rebuild_thread = spawn_rebuild_thread(
+        opt_level,
+        surgically_link,
+        precompiled,
+        host_input_path.clone(),
+        binary_path.clone(),
+        target,
+        loaded
+            .exposed_to_host
+            .keys()
+            .map(|x| x.as_str(&loaded.interns).to_string())
+            .collect(),
+    );
+
+    // TODO try to move as much of this linking as possible to the precompiled
+    // host, to minimize the amount of host-application linking required.
     let app_o_file = Builder::new()
         .prefix("roc_app")
         .suffix(&format!(".{}", app_extension))
@@ -142,13 +176,6 @@ pub fn build_file<'a>(
     program::report_problems(&mut loaded);
     let loaded = loaded;
 
-    let cwd = roc_file_path.parent().unwrap();
-    let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows
-
-    if emit_wasm {
-        binary_path.set_extension("wasm");
-    }
-
     let code_gen_timing = match opt_level {
         OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm(
             arena,
@@ -196,44 +223,45 @@ pub fn build_file<'a>(
         );
     }
 
-    // Step 2: link the precompiled host and compiled app
+    let rebuild_duration = rebuild_thread.join().unwrap();
-    let mut host_input_path = PathBuf::from(cwd);
+    if emit_timings && !precompiled {
-
-    host_input_path.push(&*path_to_platform);
-    host_input_path.push("host");
-    host_input_path.set_extension(host_extension);
-
-    // TODO we should no longer need to do this once we have platforms on
-    // a package repository, as we can then get precompiled hosts from there.
-    let rebuild_host_start = SystemTime::now();
-    rebuild_host(target, host_input_path.as_path());
-    let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
-
-    if emit_timings {
         println!(
-            "Finished rebuilding the host in {} ms\n",
+            "Finished rebuilding and preprocessing the host in {} ms\n",
-            rebuild_host_end.as_millis()
+            rebuild_duration
         );
     }
 
-    // TODO try to move as much of this linking as possible to the precompiled
+    // Step 2: link the precompiled host and compiled app
-    // host, to minimize the amount of host-application linking required.
     let link_start = SystemTime::now();
-    let (mut child, binary_path) =  // TODO use lld
+    let outcome = if surgically_link {
-        link(
+        roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path)
-            target,
+            .map_err(|_| {
-            binary_path,
+                todo!("gracefully handle failing to surgically link");
-            &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()],
+            })?;
-            link_type
+        BuildOutcome::NoProblems
-        )
+    } else {
-        .map_err(|_| {
+        let (mut child, _) =  // TODO use lld
-            todo!("gracefully handle `rustc` failing to spawn.");
+            link(
+                target,
+                binary_path.clone(),
+                &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()],
+                link_type
+            )
+            .map_err(|_| {
+                todo!("gracefully handle `ld` failing to spawn.");
+            })?;
+
+        let exit_status = child.wait().map_err(|_| {
+            todo!("gracefully handle error after `ld` spawned");
         })?;
 
-    let cmd_result = child.wait().map_err(|_| {
+        // TODO change this to report whether there were errors or warnings!
-        todo!("gracefully handle error after `rustc` spawned");
+        if exit_status.success() {
-    });
+            BuildOutcome::NoProblems
-
+        } else {
+            BuildOutcome::Errors
+        }
+    };
     let linking_time = link_start.elapsed().unwrap();
 
     if emit_timings {
@@ -242,16 +270,6 @@ pub fn build_file<'a>(
 
     let total_time = compilation_start.elapsed().unwrap();
 
-    // If the cmd errored out, return the Err.
-    let exit_status = cmd_result?;
-
-    // TODO change this to report whether there were errors or warnings!
-    let outcome = if exit_status.success() {
-        BuildOutcome::NoProblems
-    } else {
-        BuildOutcome::Errors
-    };
-
     Ok(BuiltFile {
         binary_path,
         outcome,
@@ -259,6 +277,46 @@ pub fn build_file<'a>(
     })
 }
 
+fn spawn_rebuild_thread(
+    opt_level: OptLevel,
+    surgically_link: bool,
+    precompiled: bool,
+    host_input_path: PathBuf,
+    binary_path: PathBuf,
+    target: &Triple,
+    exported_symbols: Vec<String>,
+) -> std::thread::JoinHandle<u128> {
+    let thread_local_target = target.clone();
+    std::thread::spawn(move || {
+        let rebuild_host_start = SystemTime::now();
+        if !precompiled {
+            if surgically_link {
+                roc_linker::build_and_preprocess_host(
+                    opt_level,
+                    &thread_local_target,
+                    host_input_path.as_path(),
+                    exported_symbols,
+                )
+                .unwrap();
+            } else {
+                rebuild_host(
+                    opt_level,
+                    &thread_local_target,
+                    host_input_path.as_path(),
+                    None,
+                );
+            }
+        }
+        if surgically_link {
+            // Copy preprocessed host to executable location.
+            let prehost = host_input_path.with_file_name("preprocessedhost");
+            std::fs::copy(prehost, binary_path.as_path()).unwrap();
+        }
+        let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
+        rebuild_host_end.as_millis()
+    })
+}
+
 #[allow(clippy::too_many_arguments)]
 pub fn check_file(
     arena: &Bump,

cli/src/lib.rs

@@ -34,6 +34,8 @@ pub const FLAG_OPTIMIZE: &str = "optimize";
 pub const FLAG_LIB: &str = "lib";
 pub const FLAG_BACKEND: &str = "backend";
 pub const FLAG_TIME: &str = "time";
+pub const FLAG_LINK: &str = "roc-linker";
+pub const FLAG_PRECOMPILED: &str = "precompiled-host";
 pub const ROC_FILE: &str = "ROC_FILE";
 pub const BACKEND: &str = "BACKEND";
 pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES";
@@ -89,6 +91,18 @@ pub fn build_app<'a>() -> App<'a> {
                     .help("Prints detailed compilation time information.")
                     .required(false),
             )
+            .arg(
+                Arg::with_name(FLAG_LINK)
+                    .long(FLAG_LINK)
+                    .help("Uses the roc linker instead of the system linker.")
+                    .required(false),
+            )
+            .arg(
+                Arg::with_name(FLAG_PRECOMPILED)
+                    .long(FLAG_PRECOMPILED)
+                    .help("Assumes the host has been precompiled and skips recompiling the host.")
+                    .required(false),
+            )
         )
         .subcommand(App::new(CMD_RUN)
             .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`")
@@ -177,6 +191,18 @@ pub fn build_app<'a>() -> App<'a> {
                 .help("Prints detailed compilation time information.")
                     .required(false),
         )
+        .arg(
+            Arg::with_name(FLAG_LINK)
+                .long(FLAG_LINK)
+                .help("Uses the roc linker instead of the system linker.")
+                .required(false),
+        )
+        .arg(
+            Arg::with_name(FLAG_PRECOMPILED)
+                .long(FLAG_PRECOMPILED)
+                .help("Assumes the host has been precompiled and skips recompiling the host.")
+                .required(false),
+        )
         .arg(
             Arg::with_name(FLAG_BACKEND)
                 .long(FLAG_BACKEND)
@@ -261,6 +287,14 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
     } else {
         LinkType::Executable
     };
+    let surgically_link = matches.is_present(FLAG_LINK);
+    let precompiled = matches.is_present(FLAG_PRECOMPILED);
+    if surgically_link && !roc_linker::supported(&link_type, &target) {
+        panic!(
+            "Link type, {:?}, with target, {}, not supported by roc linker",
+            link_type, target
+        );
+    }
 
     let path = Path::new(filename);
 
@@ -293,6 +327,8 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
         emit_debug_info,
         emit_timings,
         link_type,
+        surgically_link,
+        precompiled,
     );
 
     match res_binary_path {

compiler/build/src/link.rs

@@ -76,6 +76,7 @@ fn find_wasi_libc_path() -> PathBuf {
 }
 
 #[cfg(not(target_os = "macos"))]
+#[allow(clippy::too_many_arguments)]
 pub fn build_zig_host_native(
     env_path: &str,
     env_home: &str,
@@ -83,36 +84,44 @@ pub fn build_zig_host_native(
     zig_host_src: &str,
     zig_str_path: &str,
     target: &str,
+    opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
-    Command::new("zig")
+    let mut command = Command::new("zig");
+    command
         .env_clear()
         .env("PATH", env_path)
-        .env("HOME", env_home)
+        .env("HOME", env_home);
-        .args(&[
+    if let Some(shared_lib_path) = shared_lib_path {
-            "build-obj",
+        command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]);
-            zig_host_src,
+    } else {
-            emit_bin,
+        command.args(&["build-obj", "-fPIC"]);
-            "--pkg-begin",
+    }
-            "str",
+    command.args(&[
-            zig_str_path,
+        zig_host_src,
-            "--pkg-end",
+        emit_bin,
-            // include the zig runtime
+        "--pkg-begin",
-            "-fcompiler-rt",
+        "str",
-            // include libc
+        zig_str_path,
-            "--library",
+        "--pkg-end",
-            "c",
+        // include the zig runtime
-            "-fPIC",
+        "-fcompiler-rt",
-            "-O",
+        // include libc
-            "ReleaseSafe",
+        "--library",
-            // cross-compile?
+        "c",
-            "-target",
+        "--strip",
-            target,
+        // cross-compile?
-        ])
+        "-target",
-        .output()
+        target,
-        .unwrap()
+    ]);
+    if matches!(opt_level, OptLevel::Optimize) {
+        command.args(&["-O", "ReleaseSafe"]);
+    }
+    command.output().unwrap()
 }
 
 #[cfg(target_os = "macos")]
+#[allow(clippy::too_many_arguments)]
 pub fn build_zig_host_native(
     env_path: &str,
     env_home: &str,
@@ -120,6 +129,8 @@ pub fn build_zig_host_native(
     zig_host_src: &str,
     zig_str_path: &str,
     _target: &str,
+    opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
     use serde_json::Value;
 
@@ -161,32 +172,37 @@ pub fn build_zig_host_native(
     zig_compiler_rt_path.push("special");
     zig_compiler_rt_path.push("compiler_rt.zig");
 
-    Command::new("zig")
+    let mut command = Command::new("zig");
+    command
         .env_clear()
         .env("PATH", &env_path)
-        .env("HOME", &env_home)
+        .env("HOME", &env_home);
-        .args(&[
+    if let Some(shared_lib_path) = shared_lib_path {
-            "build-obj",
+        command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]);
-            zig_host_src,
+    } else {
-            emit_bin,
+        command.args(&["build-obj", "-fPIC"]);
-            "--pkg-begin",
+    }
-            "str",
+    command.args(&[
-            zig_str_path,
+        zig_host_src,
-            "--pkg-end",
+        emit_bin,
-            // include the zig runtime
+        "--pkg-begin",
-            "--pkg-begin",
+        "str",
-            "compiler_rt",
+        zig_str_path,
-            zig_compiler_rt_path.to_str().unwrap(),
+        "--pkg-end",
-            "--pkg-end",
+        // include the zig runtime
-            // include libc
+        "--pkg-begin",
-            "--library",
+        "compiler_rt",
-            "c",
+        zig_compiler_rt_path.to_str().unwrap(),
-            "-fPIC",
+        "--pkg-end",
-            "-O",
+        // include libc
-            "ReleaseSafe",
+        "--library",
-        ])
+        "c",
-        .output()
+        "--strip",
-        .unwrap()
+    ]);
+    if matches!(opt_level, OptLevel::Optimize) {
+        command.args(&["-O", "ReleaseSafe"]);
+    }
+    command.output().unwrap()
 }
 
 pub fn build_zig_host_wasm32(
@@ -195,7 +211,12 @@ pub fn build_zig_host_wasm32(
     emit_bin: &str,
     zig_host_src: &str,
     zig_str_path: &str,
+    opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
+    if shared_lib_path.is_some() {
+        unimplemented!("Linking a shared library to wasm not yet implemented");
+    }
     // NOTE currently just to get compiler warnings if the host code is invalid.
     // the produced artifact is not used
     //
@@ -204,7 +225,8 @@ pub fn build_zig_host_wasm32(
     // we'd like to compile with `-target wasm32-wasi` but that is blocked on
     //
     // https://github.com/ziglang/zig/issues/9414
-    Command::new("zig")
+    let mut command = Command::new("zig");
+    command
         .env_clear()
         .env("PATH", env_path)
         .env("HOME", env_home)
@@ -226,21 +248,66 @@ pub fn build_zig_host_wasm32(
             // "wasm32-wasi",
             // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll",
             "-fPIC",
-            "-O",
+            "--strip",
-            "ReleaseSafe",
+        ]);
-        ])
+    if matches!(opt_level, OptLevel::Optimize) {
-        .output()
+        command.args(&["-O", "ReleaseSafe"]);
-        .unwrap()
+    }
+    command.output().unwrap()
 }
 
-pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
+pub fn build_c_host_native(
+    env_path: &str,
+    env_home: &str,
+    dest: &str,
+    sources: &[&str],
+    opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
+) -> Output {
+    let mut command = Command::new("clang");
+    command
+        .env_clear()
+        .env("PATH", &env_path)
+        .env("HOME", &env_home)
+        .args(sources)
+        .args(&["-o", dest]);
+    if let Some(shared_lib_path) = shared_lib_path {
+        command.args(&[
+            shared_lib_path.to_str().unwrap(),
+            "-fPIE",
+            "-pie",
+            "-lm",
+            "-lpthread",
+            "-ldl",
+            "-lrt",
+            "-lutil",
+        ]);
+    } else {
+        command.args(&["-fPIC", "-c"]);
+    }
+    if matches!(opt_level, OptLevel::Optimize) {
+        command.arg("-O2");
+    }
+    command.output().unwrap()
+}
+
+pub fn rebuild_host(
+    opt_level: OptLevel,
+    target: &Triple,
+    host_input_path: &Path,
+    shared_lib_path: Option<&Path>,
+) {
     let c_host_src = host_input_path.with_file_name("host.c");
     let c_host_dest = host_input_path.with_file_name("c_host.o");
     let zig_host_src = host_input_path.with_file_name("host.zig");
     let rust_host_src = host_input_path.with_file_name("host.rs");
     let rust_host_dest = host_input_path.with_file_name("rust_host.o");
     let cargo_host_src = host_input_path.with_file_name("Cargo.toml");
-    let host_dest_native = host_input_path.with_file_name("host.o");
+    let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() {
+        "dynhost"
+    } else {
+        "host.o"
+    });
     let host_dest_wasm = host_input_path.with_file_name("host.bc");
 
     let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
@@ -266,6 +333,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
                     &emit_bin,
                     zig_host_src.to_str().unwrap(),
                     zig_str_path.to_str().unwrap(),
+                    opt_level,
+                    shared_lib_path,
                 )
             }
             Architecture::X86_64 => {
@@ -277,6 +346,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
                     zig_host_src.to_str().unwrap(),
                     zig_str_path.to_str().unwrap(),
                     "native",
+                    opt_level,
+                    shared_lib_path,
                 )
             }
             Architecture::X86_32(_) => {
@@ -288,89 +359,142 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
                     zig_host_src.to_str().unwrap(),
                     zig_str_path.to_str().unwrap(),
                     "i386-linux-musl",
+                    opt_level,
+                    shared_lib_path,
                 )
             }
             _ => panic!("Unsupported architecture {:?}", target.architecture),
         };
 
         validate_output("host.zig", "zig", output)
-    } else {
+    } else if cargo_host_src.exists() {
-        // Compile host.c
-        let output = Command::new("clang")
-            .env_clear()
-            .env("PATH", &env_path)
-            .args(&[
-                "-O2",
-                "-fPIC",
-                "-c",
-                c_host_src.to_str().unwrap(),
-                "-o",
-                c_host_dest.to_str().unwrap(),
-            ])
-            .output()
-            .unwrap();
-
-        validate_output("host.c", "clang", output);
-    }
-
-    if cargo_host_src.exists() {
         // Compile and link Cargo.toml, if it exists
         let cargo_dir = host_input_path.parent().unwrap();
-        let libhost_dir = cargo_dir.join("target").join("release");
+        let libhost_dir =
+            cargo_dir
+                .join("target")
+                .join(if matches!(opt_level, OptLevel::Optimize) {
+                    "release"
+                } else {
+                    "debug"
+                });
+        let libhost = libhost_dir.join("libhost.a");
 
-        let output = Command::new("cargo")
+        let mut command = Command::new("cargo");
-            .args(&["build", "--release"])
+        command.arg("build").current_dir(cargo_dir);
-            .current_dir(cargo_dir)
+        if matches!(opt_level, OptLevel::Optimize) {
-            .output()
+            command.arg("--release");
-            .unwrap();
+        }
+        let output = command.output().unwrap();
 
-        validate_output("src/lib.rs", "cargo build --release", output);
+        validate_output("src/lib.rs", "cargo build", output);
 
-        let output = Command::new("ld")
+        // Cargo hosts depend on a c wrapper for the api. Compile host.c as well.
-            .env_clear()
+        if shared_lib_path.is_some() {
-            .env("PATH", &env_path)
+            // If compiling to executable, let c deal with linking as well.
-            .args(&[
+            let output = build_c_host_native(
-                "-r",
+                &env_path,
-                "-L",
+                &env_home,
-                libhost_dir.to_str().unwrap(),
-                c_host_dest.to_str().unwrap(),
-                "-lhost",
-                "-o",
                 host_dest_native.to_str().unwrap(),
-            ])
+                &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()],
-            .output()
+                opt_level,
-            .unwrap();
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
+        } else {
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                c_host_dest.to_str().unwrap(),
+                &[c_host_src.to_str().unwrap()],
+                opt_level,
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
 
-        validate_output("c_host.o", "ld", output);
+            let output = Command::new("ld")
+                .env_clear()
+                .env("PATH", &env_path)
+                .args(&[
+                    "-r",
+                    "-L",
+                    libhost_dir.to_str().unwrap(),
+                    c_host_dest.to_str().unwrap(),
+                    "-lhost",
+                    "-o",
+                    host_dest_native.to_str().unwrap(),
+                ])
+                .output()
+                .unwrap();
+            validate_output("c_host.o", "ld", output);
+
+            // Clean up c_host.o
+            let output = Command::new("rm")
+                .env_clear()
+                .args(&["-f", c_host_dest.to_str().unwrap()])
+                .output()
+                .unwrap();
+
+            validate_output("rust_host.o", "rm", output);
+        }
     } else if rust_host_src.exists() {
         // Compile and link host.rs, if it exists
-        let output = Command::new("rustc")
+        let mut command = Command::new("rustc");
-            .args(&[
+        command.args(&[
-                rust_host_src.to_str().unwrap(),
+            rust_host_src.to_str().unwrap(),
-                "-o",
+            "-o",
-                rust_host_dest.to_str().unwrap(),
+            rust_host_dest.to_str().unwrap(),
-            ])
+        ]);
-            .output()
+        if matches!(opt_level, OptLevel::Optimize) {
-            .unwrap();
+            command.arg("-O");
+        }
+        let output = command.output().unwrap();
 
         validate_output("host.rs", "rustc", output);
 
-        let output = Command::new("ld")
+        // Rust hosts depend on a c wrapper for the api. Compile host.c as well.
-            .env_clear()
+        if shared_lib_path.is_some() {
-            .env("PATH", &env_path)
+            // If compiling to executable, let c deal with linking as well.
-            .args(&[
+            let output = build_c_host_native(
-                "-r",
+                &env_path,
-                c_host_dest.to_str().unwrap(),
+                &env_home,
-                rust_host_dest.to_str().unwrap(),
-                "-o",
                 host_dest_native.to_str().unwrap(),
-            ])
+                &[
-            .output()
+                    c_host_src.to_str().unwrap(),
-            .unwrap();
+                    rust_host_dest.to_str().unwrap(),
+                ],
+                opt_level,
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
+        } else {
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                c_host_dest.to_str().unwrap(),
+                &[c_host_src.to_str().unwrap()],
+                opt_level,
+                shared_lib_path,
+            );
 
-        validate_output("rust_host.o", "ld", output);
+            validate_output("host.c", "clang", output);
+            let output = Command::new("ld")
+                .env_clear()
+                .env("PATH", &env_path)
+                .args(&[
+                    "-r",
+                    c_host_dest.to_str().unwrap(),
+                    rust_host_dest.to_str().unwrap(),
+                    "-o",
+                    host_dest_native.to_str().unwrap(),
+                ])
+                .output()
+                .unwrap();
 
-        // Clean up rust_host.o
+            validate_output("rust_host.o", "ld", output);
+        }
+
+        // Clean up rust_host.o and c_host.o
         let output = Command::new("rm")
             .env_clear()
             .args(&[
@@ -382,15 +506,17 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
             .unwrap();
 
         validate_output("rust_host.o", "rm", output);
-    } else if c_host_dest.exists() {
+    } else if c_host_src.exists() {
-        // Clean up c_host.o
+        // Compile host.c, if it exists
-        let output = Command::new("mv")
+        let output = build_c_host_native(
-            .env_clear()
+            &env_path,
-            .args(&[c_host_dest, host_dest_native])
+            &env_home,
-            .output()
+            host_dest_native.to_str().unwrap(),
-            .unwrap();
+            &[c_host_src.to_str().unwrap()],
-
+            opt_level,
-        validate_output("c_host.o", "mv", output);
+            shared_lib_path,
+        );
+        validate_output("host.c", "clang", output);
     }
 }
 

examples/.gitignore

@@ -4,3 +4,7 @@ app
 libhost.a
 roc_app.ll
 roc_app.bc
+dynhost
+preprocessedhost
+metadata
+libapp.so

examples/benchmarks/platform/host.zig

@@ -29,10 +29,13 @@ extern fn roc__mainForHost_1_Fx_caller(*const u8, [*]u8, [*]u8) void;
 extern fn roc__mainForHost_1_Fx_size() i64;
 extern fn roc__mainForHost_1_Fx_result_size() i64;
 
+
 const Align = 2 * @alignOf(usize);
 extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void;
 extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void;
 extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void;
+extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
+extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
 
 const DEBUG: bool = false;
 
@@ -74,6 +77,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
     std.process.exit(0);
 }
 
+export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
+    return memcpy(dst, src, size);
+}
+
+export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
+    return memset(dst, value, size);
+}
+
 const Unit = extern struct {};
 
 pub export fn main() callconv(.C) u8 {

examples/effect/thing/platform-dir/host.zig

@@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64;
 extern fn malloc(size: usize) callconv(.C) ?*c_void;
 extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void;
 extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void;
+extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
+extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
 
 export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void {
     return malloc(size);
@@ -52,6 +54,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
     std.process.exit(0);
 }
 
+export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
+    return memcpy(dst, src, size);
+}
+
+export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
+    return memset(dst, value, size);
+}
+
 const Unit = extern struct {};
 
 pub export fn main() u8 {

examples/hello-rust/platform/host.c

@@ -1,7 +1,12 @@
 #include <stdio.h>
+#include <string.h>
 
 extern int rust_main();
 
-int main() {
+int main() { return rust_main(); }
-  return rust_main();
+
+void *roc_memcpy(void *dest, const void *src, size_t n) {
+  return memcpy(dest, src, n);
 }
+
+void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); }

examples/hello-world/platform/host.c

@@ -1,81 +1,81 @@
-#include <stdlib.h>
-#include <unistd.h>
-#include <stdio.h>
 #include <errno.h>
-#include <string.h>
 #include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
 
-void* roc_alloc(size_t size, unsigned int alignment) {
+void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); }
-    return malloc(size);
+
+void* roc_realloc(void* ptr, size_t old_size, size_t new_size,
+                  unsigned int alignment) {
+  return realloc(ptr, new_size);
 }
 
-void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) {
+void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); }
-    return realloc(ptr, new_size);
-}
-
-void roc_dealloc(void* ptr, unsigned int alignment) {
-    free(ptr);
-}
 
 void roc_panic(void* ptr, unsigned int alignment) {
-    char* msg = (char *)ptr; 
+  char* msg = (char*)ptr;
-    fprintf(stderr, "Application crashed with message\n\n    %s\n\nShutting down\n", msg);
+  fprintf(stderr,
-    exit(0);
+          "Application crashed with message\n\n    %s\n\nShutting down\n", msg);
+  exit(0);
 }
 
+void* roc_memcpy(void* dest, const void* src, size_t n) {
+  return memcpy(dest, src, n);
+}
+
+void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); }
+
 struct RocStr {
-    char* bytes;
+  char* bytes;
-    size_t len;
+  size_t len;
 };
 
-bool is_small_str(struct RocStr str) {
+bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; }
-    return ((ssize_t)str.len) < 0;
-}
 
 // Determine the length of the string, taking into
 // account the small string optimization
 size_t roc_str_len(struct RocStr str) {
-    char* bytes = (char*)&str;
+  char* bytes = (char*)&str;
-    char last_byte = bytes[sizeof(str) - 1];
+  char last_byte = bytes[sizeof(str) - 1];
-    char last_byte_xored = last_byte ^ 0b10000000;
+  char last_byte_xored = last_byte ^ 0b10000000;
-    size_t small_len = (size_t)(last_byte_xored);
+  size_t small_len = (size_t)(last_byte_xored);
-    size_t big_len = str.len;
+  size_t big_len = str.len;
 
-    // Avoid branch misprediction costs by always
+  // Avoid branch misprediction costs by always
-    // determining both small_len and big_len,
+  // determining both small_len and big_len,
-    // so this compiles to a cmov instruction.
+  // so this compiles to a cmov instruction.
-    if (is_small_str(str)) {
+  if (is_small_str(str)) {
-        return small_len;
+    return small_len;
-    } else {
+  } else {
-        return big_len;
+    return big_len;
-    }
+  }
 }
 
 extern struct RocStr roc__mainForHost_1_exposed();
 
 int main() {
-    // Call Roc to populate call_result
+  struct RocStr str = roc__mainForHost_1_exposed();
-    struct RocStr call_result = roc__mainForHost_1_exposed();
 
-    // Determine str_len and the str_bytes pointer,
+  // Determine str_len and the str_bytes pointer,
-    // taking into account the small string optimization.
+  // taking into account the small string optimization.
-    struct RocStr str = call_result;
+  size_t str_len = roc_str_len(str);
-    size_t str_len = roc_str_len(str);
+  char* str_bytes;
-    char* str_bytes;
 
-    if (is_small_str(str)) {
+  if (is_small_str(str)) {
-        str_bytes = (char*)&str;
+    str_bytes = (char*)&str;
-    } else {
+  } else {
-        str_bytes = str.bytes;
+    str_bytes = str.bytes;
-    }
+  }
 
-    // Write to stdout
+  // Write to stdout
-    if (write(1, str_bytes, str_len) >= 0) {
+  if (write(1, str_bytes, str_len) >= 0) {
-        // Writing succeeded!
+    // Writing succeeded!
-        return 0;
+    return 0;
-    } else {
+  } else {
-        printf("Error writing to stdout: %s\n", strerror(errno));
+    printf("Error writing to stdout: %s\n", strerror(errno));
 
-        return 1;
+    return 1;
-    }
+  }
 }

examples/hello-zig/platform/host.zig

@@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize };
 extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void;
 extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void;
 extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void;
+extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
+extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
 
 export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void {
     _ = alignment;
@@ -51,6 +53,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
     std.process.exit(0);
 }
 
+export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
+    return memcpy(dst, src, size);
+}
+
+export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
+    return memset(dst, value, size);
+}
+
 const mem = std.mem;
 const Allocator = mem.Allocator;
 

examples/quicksort/platform/host.zig

@@ -26,6 +26,8 @@ const Align = extern struct { a: usize, b: usize };
 extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void;
 extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void;
 extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void;
+extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
+extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
 
 const DEBUG: bool = false;
 
@@ -67,6 +69,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
     std.process.exit(0);
 }
 
+export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
+    return memcpy(dst, src, size);
+}
+
+export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
+    return memset(dst, value, size);
+}
+
 // warning! the array is currently stack-allocated so don't make this too big
 const NUM_NUMS = 100;
 

linker/Cargo.toml

@@ -18,12 +18,16 @@ test = false
 bench = false
 
 [dependencies]
+roc_mono = { path = "../compiler/mono" }
+roc_build = { path = "../compiler/build", default-features = false }
 roc_collections = { path = "../compiler/collections" }
 bumpalo = { version = "3.6", features = ["collections"] }
 # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
 clap = { git = "https://github.com/rtfeldman/clap", branch = "master" }
 iced-x86 = "1.14"
 memmap2 = "0.3"
-object = { version = "0.26", features = ["read"] }
+object = { version = "0.26", features = ["read", "write"] }
 serde = { version = "1.0", features = ["derive"] }
 bincode = "1.3"
+target-lexicon = "0.12.2"
+tempfile = "3.1.0"

linker/README.md

@@ -29,13 +29,16 @@ This linker is run in 2 phases: preprocessing and surigical linking.
 1. Surgically update all call locations in the platform
 1. Surgically update call information in the application (also dealing with other relocations for builtins)
 
-## TODO for merging with compiler flow
+## TODO (In a lightly prioritized order)
 
-1. Add new compiler flag to hide this all behind.
+- Run CLI tests and/or benchmarks with the Roc Linker.
-1. Get compiler to generate dummy shared libraries with Roc exported symbols defined.
+- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig).
-1. Modify host linking to generate dynamic executable that links against the dummy lib.
+- Add Macho support
-1. Call the preprocessor on the dynamic executable host.
+  - Honestly should be almost exactly the same code.
-1. Call the surgical linker on the emitted roc object file and the preprocessed host.
+    This means we likely need to do a lot of refactoring to minimize the duplicate code.
-1. Enjoy!
+    The fun of almost but not quite the same.
-1. Extract preprocessing generation to run earlier, maybe in parallel with the main compiler until we have full precompiled hosts.
+- Add PE support
-1. Maybe add a roc command to generate the dummy lib to be used by platform authors.
+  - As a prereq, we need roc building on Windows (I'm not sure it does currently).
+  - Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add.
+- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl`
+- Add a feature to the compiler to make this linker optional.

linker/src/lib.rs

@@ -2,13 +2,16 @@ use bincode::{deserialize_from, serialize_into};
 use clap::{App, AppSettings, Arg, ArgMatches};
 use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind};
 use memmap2::{Mmap, MmapMut};
+use object::write;
 use object::{elf, endian};
 use object::{
-    Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, LittleEndian, NativeEndian,
+    Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian,
-    Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
+    NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section,
-    Symbol, SymbolIndex, SymbolSection,
+    SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection,
 };
+use roc_build::link::{rebuild_host, LinkType};
 use roc_collections::all::MutMap;
+use roc_mono::ir::OptLevel;
 use std::cmp::Ordering;
 use std::convert::TryFrom;
 use std::ffi::CStr;
@@ -17,8 +20,12 @@ use std::io;
 use std::io::{BufReader, BufWriter};
 use std::mem;
 use std::os::raw::c_char;
+use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
+use std::process::Command;
 use std::time::{Duration, SystemTime};
+use target_lexicon::Triple;
+use tempfile::Builder;
 
 mod metadata;
 
@@ -122,15 +129,149 @@ pub fn build_app<'a>() -> App<'a> {
         )
 }
 
+pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
+    link_type == &LinkType::Executable
+        && target.architecture == target_lexicon::Architecture::X86_64
+        && target.operating_system == target_lexicon::OperatingSystem::Linux
+        && target.binary_format == target_lexicon::BinaryFormat::Elf
+}
+
+pub fn build_and_preprocess_host(
+    opt_level: OptLevel,
+    target: &Triple,
+    host_input_path: &Path,
+    exposed_to_host: Vec<String>,
+) -> io::Result<()> {
+    let dummy_lib = host_input_path.with_file_name("libapp.so");
+    generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?;
+    rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib));
+    let dynhost = host_input_path.with_file_name("dynhost");
+    let metadata = host_input_path.with_file_name("metadata");
+    let prehost = host_input_path.with_file_name("preprocessedhost");
+    if preprocess_impl(
+        dynhost.to_str().unwrap(),
+        metadata.to_str().unwrap(),
+        prehost.to_str().unwrap(),
+        dummy_lib.to_str().unwrap(),
+        false,
+        false,
+    )? != 0
+    {
+        panic!("Failed to preprocess host");
+    }
+    Ok(())
+}
+
+pub fn link_preprocessed_host(
+    _target: &Triple,
+    host_input_path: &Path,
+    roc_app_obj: &Path,
+    binary_path: &Path,
+) -> io::Result<()> {
+    let metadata = host_input_path.with_file_name("metadata");
+    if surgery_impl(
+        roc_app_obj.to_str().unwrap(),
+        metadata.to_str().unwrap(),
+        binary_path.to_str().unwrap(),
+        false,
+        false,
+    )? != 0
+    {
+        panic!("Failed to surgically link host");
+    }
+    Ok(())
+}
+
+fn generate_dynamic_lib(
+    _target: &Triple,
+    exposed_to_host: Vec<String>,
+    dummy_lib_path: &Path,
+) -> io::Result<()> {
+    let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?;
+    let dummy_obj_file = dummy_obj_file.path();
+
+    // TODO deal with other architectures here.
+    let mut out_object =
+        write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little);
+
+    let text_section = out_object.section_id(write::StandardSection::Text);
+    for sym in exposed_to_host {
+        // TODO properly generate this list.
+        for name in &[
+            format!("roc__{}_1_exposed", sym),
+            format!("roc__{}_1_Fx_caller", sym),
+            format!("roc__{}_1_Fx_size", sym),
+            format!("roc__{}_1_Fx_result_size", sym),
+            format!("roc__{}_size", sym),
+        ] {
+            out_object.add_symbol(write::Symbol {
+                name: name.as_bytes().to_vec(),
+                value: 0,
+                size: 0,
+                kind: SymbolKind::Text,
+                scope: SymbolScope::Dynamic,
+                weak: false,
+                section: write::SymbolSection::Section(text_section),
+                flags: SymbolFlags::None,
+            });
+        }
+    }
+    std::fs::write(
+        &dummy_obj_file,
+        out_object.write().expect("failed to build output object"),
+    )
+    .expect("failed to write object to file");
+
+    let output = Command::new("ld")
+        .args(&[
+            "-shared",
+            "-soname",
+            dummy_lib_path.file_name().unwrap().to_str().unwrap(),
+            dummy_obj_file.to_str().unwrap(),
+            "-o",
+            dummy_lib_path.to_str().unwrap(),
+        ])
+        .output()
+        .unwrap();
+
+    if !output.status.success() {
+        match std::str::from_utf8(&output.stderr) {
+            Ok(stderr) => panic!(
+                "Failed to link dummy shared library - stderr of the `ld` command was:\n{}",
+                stderr
+            ),
+            Err(utf8_err) => panic!(
+                "Failed to link dummy shared library  - stderr of the `ld` command was invalid utf8 ({:?})",
+                utf8_err
+            ),
+        }
+    }
+    Ok(())
+}
+
+pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
+    preprocess_impl(
+        matches.value_of(EXEC).unwrap(),
+        matches.value_of(METADATA).unwrap(),
+        matches.value_of(OUT).unwrap(),
+        matches.value_of(SHARED_LIB).unwrap(),
+        matches.is_present(FLAG_VERBOSE),
+        matches.is_present(FLAG_TIME),
+    )
+}
 // TODO: Most of this file is a mess of giant functions just to check if things work.
 // Clean it all up and refactor nicely.
-pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
+fn preprocess_impl(
-    let verbose = matches.is_present(FLAG_VERBOSE);
+    exec_filename: &str,
-    let time = matches.is_present(FLAG_TIME);
+    metadata_filename: &str,
-
+    out_filename: &str,
+    shared_lib_filename: &str,
+    verbose: bool,
+    time: bool,
+) -> io::Result<i32> {
     let total_start = SystemTime::now();
     let exec_parsing_start = total_start;
-    let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?;
+    let exec_file = fs::File::open(exec_filename)?;
     let exec_mmap = unsafe { Mmap::map(&exec_file)? };
     let exec_data = &*exec_mmap;
     let exec_obj = match object::File::parse(exec_data) {
@@ -226,6 +367,9 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
         println!("PLT File Offset: {:+x}", plt_offset);
     }
 
+    // TODO: it looks like we may need to support global data host relocations.
+    // Rust host look to be using them by default instead of the plt.
+    // I think this is due to first linking into a static lib and then linking to the c wrapper.
     let plt_relocs = (match exec_obj.dynamic_relocations() {
         Some(relocs) => relocs,
         None => {
@@ -410,6 +554,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
                         || inst.is_jmp_far_indirect()
                         || inst.is_jmp_near_indirect())
                         && !indirect_warning_given
+                        && verbose
                     {
                         indirect_warning_given = true;
                         println!();
@@ -467,7 +612,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
         }
     };
 
-    let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap())
+    let shared_lib_name = Path::new(shared_lib_filename)
         .file_name()
         .unwrap()
         .to_str()
@@ -494,7 +639,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
             ) as usize;
             let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8;
             let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap();
-            if c_str == shared_lib_name {
+            if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name {
                 shared_lib_index = Some(dyn_lib_index);
                 if verbose {
                     println!(
@@ -601,7 +746,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
         .write(true)
         .create(true)
         .truncate(true)
-        .open(&matches.value_of(OUT).unwrap())?;
+        .open(out_filename)?;
     out_file.set_len(md.exec_len)?;
     let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? };
 
@@ -862,16 +1007,22 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
     }
 
     let saving_metadata_start = SystemTime::now();
-    let output = fs::File::create(&matches.value_of(METADATA).unwrap())?;
+    // This block ensure that the metadata is fully written and timed before continuing.
-    let output = BufWriter::new(output);
+    {
-    if let Err(err) = serialize_into(output, &md) {
+        let output = fs::File::create(metadata_filename)?;
-        println!("Failed to serialize metadata: {}", err);
+        let output = BufWriter::new(output);
-        return Ok(-1);
+        if let Err(err) = serialize_into(output, &md) {
-    };
+            println!("Failed to serialize metadata: {}", err);
+            return Ok(-1);
+        };
+    }
     let saving_metadata_duration = saving_metadata_start.elapsed().unwrap();
 
     let flushing_data_start = SystemTime::now();
     out_mmap.flush()?;
+    // Also drop files to to ensure data is fully written here.
+    drop(out_mmap);
+    drop(out_file);
     let flushing_data_duration = flushing_data_start.elapsed().unwrap();
 
     let total_duration = total_start.elapsed().unwrap();
@@ -907,12 +1058,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
 }
 
 pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
-    let verbose = matches.is_present(FLAG_VERBOSE);
+    surgery_impl(
-    let time = matches.is_present(FLAG_TIME);
+        matches.value_of(APP).unwrap(),
+        matches.value_of(METADATA).unwrap(),
+        matches.value_of(OUT).unwrap(),
+        matches.is_present(FLAG_VERBOSE),
+        matches.is_present(FLAG_TIME),
+    )
+}
 
+fn surgery_impl(
+    app_filename: &str,
+    metadata_filename: &str,
+    out_filename: &str,
+    verbose: bool,
+    time: bool,
+) -> io::Result<i32> {
     let total_start = SystemTime::now();
     let loading_metadata_start = total_start;
-    let input = fs::File::open(&matches.value_of(METADATA).unwrap())?;
+    let input = fs::File::open(metadata_filename)?;
     let input = BufReader::new(input);
     let md: metadata::Metadata = match deserialize_from(input) {
         Ok(data) => data,
@@ -924,7 +1088,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
     let loading_metadata_duration = loading_metadata_start.elapsed().unwrap();
 
     let app_parsing_start = SystemTime::now();
-    let app_file = fs::File::open(&matches.value_of(APP).unwrap())?;
+    let app_file = fs::File::open(app_filename)?;
     let app_mmap = unsafe { Mmap::map(&app_file)? };
     let app_data = &*app_mmap;
     let app_obj = match object::File::parse(app_data) {
@@ -940,7 +1104,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
     let exec_file = fs::OpenOptions::new()
         .read(true)
         .write(true)
-        .open(&matches.value_of(OUT).unwrap())?;
+        .open(out_filename)?;
 
     let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint;
     exec_file.set_len(max_out_len)?;
@@ -1378,9 +1542,17 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
 
     let flushing_data_start = SystemTime::now();
     exec_mmap.flush()?;
+    // Also drop files to to ensure data is fully written here.
+    drop(exec_mmap);
+    exec_file.set_len(offset as u64 + 1)?;
+    drop(exec_file);
     let flushing_data_duration = flushing_data_start.elapsed().unwrap();
 
-    exec_file.set_len(offset as u64 + 1)?;
+    // Make sure the final executable has permision to execute.
+    let mut perms = fs::metadata(out_filename)?.permissions();
+    perms.set_mode(perms.mode() | 0o111);
+    fs::set_permissions(out_filename, perms)?;
+
     let total_duration = total_start.elapsed().unwrap();
 
     if verbose || time {

linker/tests/fib/.gitignore

@@ -4,3 +4,8 @@ zig-cache
 zig-out
 
 *.o
+
+dynhost
+preprocessedhost
+metadata
+libapp.so