Use include_bytes! so builtin hosts live in binary

Cargo.lock

@@ -3993,6 +3993,7 @@ dependencies = [
  "roc_target",
  "roc_types",
  "roc_utils",
+ "tempfile",
  "wasi_libc_sys",
  "wasm-bindgen",
  "wasm-bindgen-futures",

crates/cli/src/build.rs

@@ -345,10 +345,12 @@ pub fn build_file<'a>(
                 app_o_file.to_str().unwrap(),
             ];
 
-            let str_host_obj_path = bitcode::get_builtins_host_obj_path();
+            let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+            std::fs::write(builtins_host_file.path(), bitcode::HOST_UNIX)
+                .expect("failed to write host builtins object to tempfile");
 
             if matches!(code_gen_options.backend, program::CodeGenBackend::Assembly) {
-                inputs.push(&str_host_obj_path);
+                inputs.push(builtins_host_file.path().to_str().unwrap());
             }
 
             let (mut child, _) =  // TODO use lld

crates/compiler/build/src/link.rs

@@ -118,18 +118,24 @@ pub fn build_zig_host_native(
         // with LLVM, the builtins are already part of the roc app,
         // but with the dev backend, they are missing. To minimize work,
         // we link them as part of the host executable
-        let builtins_obj = if target.contains("windows") {
-            bitcode::get_builtins_windows_obj_path()
+        let builtins_bytes = if target.contains("windows") {
+            bitcode::HOST_WINDOWS
         } else {
-            bitcode::get_builtins_host_obj_path()
+            bitcode::HOST_UNIX
         };
 
+        // TODO in the future when we have numbered releases, this
+        // can go in ~/.cache/roc instead of writing it to a tempdir every time.
+        let builtins_host_file = tempfile::tempfile().unwrap();
+        std::fs::write(builtins_host_file.path(), builtins_bytes)
+            .expect("failed to write host builtins object to tempfile");
+
         zig_cmd.args([
             "build-exe",
             "-fPIE",
             "-rdynamic", // make sure roc_alloc and friends are exposed
             shared_lib_path.to_str().unwrap(),
-            &builtins_obj,
+            builtins_host_file.path(),
         ]);
     } else {
         zig_cmd.args(["build-obj", "-fPIC"]);
@@ -292,11 +298,22 @@ pub fn build_zig_host_native(
         .env("PATH", &env_path)
         .env("HOME", &env_home);
     if let Some(shared_lib_path) = shared_lib_path {
+        let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+
+        #[cfg(windows)]
+        let native_bitcode = bitcode::HOST_WINDOWS;
+
+        #[cfg(not(windows))]
+        let native_bitcode = bitcode::HOST_UNIX;
+
+        std::fs::write(builtins_host_file.path(), native_bitcode)
+            .expect("failed to write host builtins object to tempfile");
+
         zig_cmd.args(&[
             "build-exe",
             "-fPIE",
             shared_lib_path.to_str().unwrap(),
-            &bitcode::get_builtins_host_obj_path(),
+            builtins_host_file.path().to_str().unwrap(),
         ]);
     } else {
         zig_cmd.args(&["build-obj"]);
@@ -429,6 +446,10 @@ pub fn build_c_host_native(
                 );
             }
             _ => {
+                let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+                std::fs::write(builtins_host_file.path(), bitcode::HOST_UNIX)
+                    .expect("failed to write host builtins object to tempfile");
+
                 clang_cmd.args([
                     shared_lib_path.to_str().unwrap(),
                     // This line is commented out because
@@ -436,7 +457,7 @@ pub fn build_c_host_native(
                     // linking the built-ins led to a surgical linker bug for
                     // optimized builds. Disabling until it is needed for dev
                     // builds.
-                    // &bitcode::get_builtins_host_obj_path(),
+                    // builtins_host_file.path().to_str().unwrap(),
                     "-fPIE",
                     "-pie",
                     "-lm",
@@ -1353,10 +1374,14 @@ pub fn preprocess_host_wasm32(host_input_path: &Path, preprocessed_host_path: &P
             (but seems to be an unofficial API)
     */
 
+    let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+    std::fs::write(builtins_host_file.path(), bitcode::HOST_WASM)
+        .expect("failed to write host builtins object to tempfile");
+
     let mut zig_cmd = zig();
     let args = &[
         "wasm-ld",
-        &bitcode::get_builtins_wasm32_obj_path(),
+        builtins_host_file.path().to_str().unwrap(),
         host_input,
         WASI_LIBC_PATH,
         WASI_COMPILER_RT_PATH, // builtins need __multi3, __udivti3, __fixdfti

crates/compiler/builtins/build.rs

@@ -121,7 +121,7 @@ fn generate_bc_file(bitcode_path: &Path, zig_object: &str, file_name: &str) {
 
     // workaround for github.com/ziglang/zig/issues/9711
     #[cfg(target_os = "macos")]
-    let _ = fs::remove_dir_all("./bitcode/zig-cache");
+    let _ = fs::remove_dir_all(bitcode_path.join("zig-cache"));
 
     let mut zig_cmd = zig();
 
@@ -134,24 +134,17 @@ fn generate_bc_file(bitcode_path: &Path, zig_object: &str, file_name: &str) {
 
 pub fn get_lib_dir() -> PathBuf {
     // Currently we have the OUT_DIR variable which points to `/target/debug/build/roc_builtins-*/out/`.
-    // So we just need to shed a 3 of the outer layers to get `/target/debug/` and then add `lib`.
-    let out_dir = env::var_os("OUT_DIR").unwrap();
+    // So we just need to add "/bitcode" to that.
+    let dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()).join("bitcode");
 
-    let lib_path = Path::new(&out_dir)
-        .parent()
-        .and_then(|path| path.parent())
-        .and_then(|path| path.parent())
-        .unwrap()
-        .join("lib");
+    // create dir if it does not exist
+    fs::create_dir_all(&dir).expect("Failed to make lib dir.");
 
-    // create dir of it does not exist
-    fs::create_dir_all(lib_path.clone()).expect("Failed to make lib dir.");
-
-    lib_path
+    dir
 }
 
 fn copy_zig_builtins_to_target_dir(bitcode_path: &Path) {
-    // To enable roc to find the zig biultins, we want them to be moved to a folder next to the roc executable.
+    // To enable roc to find the zig builtins, we want them to be moved to a folder next to the roc executable.
     // So if <roc_folder>/roc is the executable. The zig files will be in <roc_folder>/lib/*.zig
     let target_profile_dir = get_lib_dir();
 

crates/compiler/builtins/src/bitcode.rs

@@ -1,40 +1,17 @@
 use roc_module::symbol::Symbol;
 use roc_target::TargetInfo;
-use roc_utils::get_lib_path;
 use std::ops::Index;
 
-const LIB_DIR_ERROR: &str = "Failed to find the lib directory. Did you copy the roc binary without also copying the lib directory?\nIf you built roc from source, the lib dir should be in target/release.\nIf not, the lib dir should be included in the release tar.gz file.";
-
-pub fn get_builtins_host_obj_path() -> String {
-    let builtins_host_path = get_lib_path().expect(LIB_DIR_ERROR).join("builtins-host.o");
-
-    builtins_host_path
-        .into_os_string()
-        .into_string()
-        .expect("Failed to convert builtins_host_path to str")
-}
-
-pub fn get_builtins_windows_obj_path() -> String {
-    let builtins_host_path = get_lib_path()
-        .expect(LIB_DIR_ERROR)
-        .join("builtins-windows-x86_64.obj");
-
-    builtins_host_path
-        .into_os_string()
-        .into_string()
-        .expect("Failed to convert builtins_host_path to str")
-}
-
-pub fn get_builtins_wasm32_obj_path() -> String {
-    let builtins_wasm32_path = get_lib_path()
-        .expect(LIB_DIR_ERROR)
-        .join("builtins-wasm32.o");
-
-    builtins_wasm32_path
-        .into_os_string()
-        .into_string()
-        .expect("Failed to convert builtins_wasm32_path to str")
-}
+pub const HOST_WASM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/bitcode/builtins-wasm32.o"));
+// TODO: in the future, we should use Zig's cross-compilation to generate and store these
+// for all targets, so that we can do cross-compilation!
+#[cfg(unix)]
+pub const HOST_UNIX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/bitcode/builtins-host.o"));
+#[cfg(windows)]
+pub const HOST_WINDOWS: &[u8] = include_bytes!(concat!(
+    env!("OUT_DIR"),
+    "/bitcode/builtins-windows-x86_64.obj"
+));
 
 #[derive(Debug, Default, Copy, Clone)]
 pub struct IntrinsicName {

crates/compiler/test_gen/Cargo.toml

@@ -14,6 +14,7 @@ path = "src/tests.rs"
 roc_builtins = { path = "../builtins" }
 roc_utils = { path = "../../utils" }
 wasi_libc_sys = { path = "../../wasi-libc-sys" }
+tempfile.workspace = true
 
 [dev-dependencies]
 roc_gen_llvm = { path = "../gen_llvm" }

crates/compiler/test_gen/build.rs

@@ -100,9 +100,13 @@ fn build_wasm_test_host() {
     let mut outfile = PathBuf::from(&out_dir).join(PLATFORM_FILENAME);
     outfile.set_extension("wasm");
 
+    let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+    std::fs::write(builtins_host_file.path(), bitcode::HOST_WASM)
+        .expect("failed to write host builtins object to tempfile");
+
     run_zig(&[
         "wasm-ld",
-        &bitcode::get_builtins_wasm32_obj_path(),
+        builtins_host_file.path().to_str().unwrap(),
         platform_path.to_str().unwrap(),
         WASI_COMPILER_RT_PATH,
         WASI_LIBC_PATH,

crates/compiler/test_gen/src/helpers/dev.rs

@@ -193,17 +193,16 @@ pub fn helper(
         .expect("failed to build output object");
     std::fs::write(&app_o_file, module_out).expect("failed to write object to file");
 
-    // std::fs::copy(&app_o_file, "/tmp/app.o").unwrap();
+    let builtins_host_file = tempfile::tempfile().unwrap();
+    std::fs::write(builtins_host_file.path(), bitcode::HOST_UNIX)
+        .expect("failed to write host builtins object to tempfile");
 
     let (mut child, dylib_path) = link(
         &target,
         app_o_file.clone(),
         // Long term we probably want a smarter way to link in zig builtins.
         // With the current method all methods are kept and it adds about 100k to all outputs.
-        &[
-            app_o_file.to_str().unwrap(),
-            &bitcode::get_builtins_host_obj_path(),
-        ],
+        &[app_o_file.to_str().unwrap(), &builtins_host_file.path()],
         LinkType::Dylib,
     )
     .expect("failed to link dynamic library");

crates/repl_wasm/Cargo.toml

@@ -13,6 +13,7 @@ crate-type = ["cdylib"]
 roc_builtins = {path = "../compiler/builtins"}
 roc_utils = {path = "../utils"}
 wasi_libc_sys = { path = "../wasi-libc-sys" }
+tempfile.workspace = true
 
 [dependencies]
 bumpalo.workspace = true

crates/repl_wasm/build.rs

@@ -23,10 +23,14 @@ fn main() {
     pre_linked_binary_path.extend(["pre_linked_binary"]);
     pre_linked_binary_path.set_extension("o");
 
+    let builtins_host_file = tempfile::NamedTempFile::new().unwrap();
+    std::fs::write(builtins_host_file.path(), bitcode::HOST_WASM)
+        .expect("failed to write host builtins object to tempfile");
+
     let output = Command::new(&zig_executable())
         .args([
             "wasm-ld",
-            &bitcode::get_builtins_wasm32_obj_path(),
+            builtins_host_file.path().to_str().unwrap(),
             platform_obj.to_str().unwrap(),
             WASI_COMPILER_RT_PATH,
             WASI_LIBC_PATH,

crates/utils/src/lib.rs

@@ -102,7 +102,7 @@ pub fn first_last_index_of<T: ::std::fmt::Debug + std::cmp::Eq>(
 }
 
 // get the path of the lib folder
-// runtime dependencies like zig files, builtin_host.o are put in the lib folder
+// runtime dependencies like zig files, Windows dylib builds, are put in the lib folder
 pub fn get_lib_path() -> Option<PathBuf> {
     let exe_relative_str_path_opt = std::env::current_exe().ok();