diff --git a/Cargo.lock b/Cargo.lock index 6819b9a48d..c17cb03261 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3728,8 +3728,10 @@ dependencies = [ "object 0.26.2", "roc_build", "roc_collections", + "roc_mono", "serde", "target-lexicon", + "tempfile", ] [[package]] diff --git a/cli/src/build.rs b/cli/src/build.rs index 3426bdad55..fec21f420b 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -284,13 +284,19 @@ fn spawn_rebuild_thread( let rebuild_host_start = SystemTime::now(); if surgically_link { roc_linker::build_and_preprocess_host( + opt_level, &thread_local_target, host_input_path.as_path(), exported_symbols, ) .unwrap(); } else { - rebuild_host(opt_level, &thread_local_target, host_input_path.as_path()); + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); } let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); rebuild_host_end.as_millis() diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 08dd7f0f6e..d6de4b12d9 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -84,30 +84,35 @@ pub fn build_zig_host_native( zig_str_path: &str, target: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { let mut command = Command::new("zig"); command .env_clear() .env("PATH", env_path) - .env("HOME", env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "-fcompiler-rt", - // include libc - "--library", - "c", - "-fPIC", - // cross-compile? - "-target", - target, - ]); + .env("HOME", env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + } else { + command.arg("build-obj"); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "-fcompiler-rt", + // include libc + "--library", + "c", + "-fPIC", + // cross-compile? + "-target", + target, + ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); } @@ -123,6 +128,7 @@ pub fn build_zig_host_native( zig_str_path: &str, _target: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { use serde_json::Value; @@ -168,25 +174,29 @@ pub fn build_zig_host_native( command .env_clear() .env("PATH", &env_path) - .env("HOME", &env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "--pkg-begin", - "compiler_rt", - zig_compiler_rt_path.to_str().unwrap(), - "--pkg-end", - // include libc - "--library", - "c", - "-fPIC", - ]); + .env("HOME", &env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", shared_lib_path.to_str().unwrap()]); + } else { + command.arg("build-obj"); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "--pkg-begin", + "compiler_rt", + zig_compiler_rt_path.to_str().unwrap(), + "--pkg-end", + // include libc + "--library", + "c", + "-fPIC", + ]); if matches!(opt_level, OptLevel::Optimize) { command.args(&["-O", "ReleaseSafe"]); } @@ -200,7 +210,11 @@ pub fn build_zig_host_wasm32( zig_host_src: &str, zig_str_path: &str, opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { + if shared_lib_path.is_some() { + unimplemented!("Linking a shared library to wasm not yet implemented"); + } // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used // @@ -239,14 +253,56 @@ pub fn build_zig_host_wasm32( command.output().unwrap() } -pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) { +pub fn build_c_host_native( + env_path: &str, + env_home: &str, + dest: &str, + sources: &[&str], + opt_level: OptLevel, + shared_lib_path: Option<&Path>, +) -> Output { + let mut command = Command::new("clang"); + command + .env_clear() + .env("PATH", &env_path) + .env("HOME", &env_home) + .args(sources) + .args(&["-fPIC", "-o", dest]); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + shared_lib_path.to_str().unwrap(), + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + ]); + } else { + command.arg("-c"); + } + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + command.output().unwrap() +} + +pub fn rebuild_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + shared_lib_path: Option<&Path>, +) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); - let host_dest_native = host_input_path.with_file_name("host.o"); + let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() { + "dynhost" + } else { + "host.o" + }); let host_dest_wasm = host_input_path.with_file_name("host.bc"); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); @@ -273,6 +329,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), opt_level, + shared_lib_path, ) } Architecture::X86_64 => { @@ -285,6 +342,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_str_path.to_str().unwrap(), "native", opt_level, + shared_lib_path, ) } Architecture::X86_32(_) => { @@ -297,31 +355,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path zig_str_path.to_str().unwrap(), "i386-linux-musl", opt_level, + shared_lib_path, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), }; validate_output("host.zig", "zig", output) - } else { - // Compile host.c - let mut command = Command::new("clang"); - command.env_clear().env("PATH", &env_path).args(&[ - "-fPIC", - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]); - if matches!(opt_level, OptLevel::Optimize) { - command.arg("-O2"); - } - let output = command.output().unwrap(); - - validate_output("host.c", "clang", output); - } - - if cargo_host_src.exists() { + } else if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); let libhost_dir = @@ -332,6 +373,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path } else { "debug" }); + let libhost = libhost_dir.join("libhost.a"); let mut command = Command::new("cargo"); command.arg("build").current_dir(cargo_dir); @@ -342,22 +384,54 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path validate_output("src/lib.rs", "cargo build", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - "-L", - libhost_dir.to_str().unwrap(), - c_host_dest.to_str().unwrap(), - "-lhost", - "-o", + // Cargo hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); - validate_output("c_host.o", "ld", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + "-L", + libhost_dir.to_str().unwrap(), + c_host_dest.to_str().unwrap(), + "-lhost", + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); + validate_output("c_host.o", "ld", output); + + // Clean up c_host.o + let output = Command::new("rm") + .env_clear() + .args(&["-f", c_host_dest.to_str().unwrap()]) + .output() + .unwrap(); + + validate_output("rust_host.o", "rm", output); + } } else if rust_host_src.exists() { // Compile and link host.rs, if it exists let mut command = Command::new("rustc"); @@ -373,22 +447,49 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path validate_output("host.rs", "rustc", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - c_host_dest.to_str().unwrap(), - rust_host_dest.to_str().unwrap(), - "-o", + // Rust hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[ + c_host_src.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + ], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); - validate_output("rust_host.o", "ld", output); + validate_output("host.c", "clang", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + c_host_dest.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); - // Clean up rust_host.o + validate_output("rust_host.o", "ld", output); + } + + // Clean up rust_host.o and c_host.o let output = Command::new("rm") .env_clear() .args(&[ @@ -400,15 +501,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path .unwrap(); validate_output("rust_host.o", "rm", output); - } else if c_host_dest.exists() { - // Clean up c_host.o - let output = Command::new("mv") - .env_clear() - .args(&[c_host_dest, host_dest_native]) - .output() - .unwrap(); - - validate_output("c_host.o", "mv", output); + } else if c_host_src.exists() { + // Compile host.c, if it exists + let output = build_c_host_native( + &env_path, + &env_home, + host_dest_native.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); } } diff --git a/linker/Cargo.toml b/linker/Cargo.toml index 520ed8f660..b5babe9ebf 100644 --- a/linker/Cargo.toml +++ b/linker/Cargo.toml @@ -18,6 +18,7 @@ test = false bench = false [dependencies] +roc_mono = { path = "../compiler/mono" } roc_build = { path = "../compiler/build", default-features = false } roc_collections = { path = "../compiler/collections" } bumpalo = { version = "3.6", features = ["collections"] } @@ -29,3 +30,4 @@ object = { version = "0.26", features = ["read"] } serde = { version = "1.0", features = ["derive"] } bincode = "1.3" target-lexicon = "0.12.2" +tempfile = "3.1.0" diff --git a/linker/src/lib.rs b/linker/src/lib.rs index c747d184f8..26eff91133 100644 --- a/linker/src/lib.rs +++ b/linker/src/lib.rs @@ -8,8 +8,9 @@ use object::{ Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, Symbol, SymbolIndex, SymbolSection, }; -use roc_build::link::LinkType; +use roc_build::link::{rebuild_host, LinkType}; use roc_collections::all::MutMap; +use roc_mono::ir::OptLevel; use std::cmp::Ordering; use std::convert::TryFrom; use std::ffi::CStr; @@ -21,6 +22,7 @@ use std::os::raw::c_char; use std::path::Path; use std::time::{Duration, SystemTime}; use target_lexicon::Triple; +use tempfile::{Builder, NamedTempFile}; mod metadata; @@ -132,27 +134,47 @@ pub fn supported(link_type: &LinkType, target: &Triple) -> bool { } pub fn build_and_preprocess_host( + opt_level: OptLevel, target: &Triple, host_input_path: &Path, exposed_to_host: Vec, ) -> io::Result<()> { let lib = generate_dynamic_lib(exposed_to_host)?; + rebuild_host(opt_level, target, host_input_path, Some(&lib.path())); Ok(()) } -fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result<()> { - Ok(()) +fn generate_dynamic_lib(exposed_to_host: Vec) -> io::Result { + for sym in exposed_to_host { + println!("{}", sym); + } + let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?; + Ok(dummy_lib_file) } +pub fn preprocess(matches: &ArgMatches) -> io::Result { + preprocess_impl( + &matches.value_of(EXEC).unwrap(), + &matches.value_of(METADATA).unwrap(), + &matches.value_of(OUT).unwrap(), + &matches.value_of(SHARED_LIB).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} // TODO: Most of this file is a mess of giant functions just to check if things work. // Clean it all up and refactor nicely. -pub fn preprocess(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); - +fn preprocess_impl( + exec_filename: &str, + metadata_filename: &str, + out_filename: &str, + shared_lib_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let exec_parsing_start = total_start; - let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?; + let exec_file = fs::File::open(exec_filename)?; let exec_mmap = unsafe { Mmap::map(&exec_file)? }; let exec_data = &*exec_mmap; let exec_obj = match object::File::parse(exec_data) { @@ -489,7 +511,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } }; - let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap()) + let shared_lib_name = Path::new(shared_lib_filename) .file_name() .unwrap() .to_str() @@ -623,7 +645,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { .write(true) .create(true) .truncate(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; out_file.set_len(md.exec_len)?; let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; @@ -884,7 +906,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } let saving_metadata_start = SystemTime::now(); - let output = fs::File::create(&matches.value_of(METADATA).unwrap())?; + let output = fs::File::create(metadata_filename)?; let output = BufWriter::new(output); if let Err(err) = serialize_into(output, &md) { println!("Failed to serialize metadata: {}", err); @@ -929,12 +951,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result { } pub fn surgery(matches: &ArgMatches) -> io::Result { - let verbose = matches.is_present(FLAG_VERBOSE); - let time = matches.is_present(FLAG_TIME); + surgery_impl( + &matches.value_of(APP).unwrap(), + &matches.value_of(METADATA).unwrap(), + &matches.value_of(OUT).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} +fn surgery_impl( + app_filename: &str, + metadata_filename: &str, + out_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { let total_start = SystemTime::now(); let loading_metadata_start = total_start; - let input = fs::File::open(&matches.value_of(METADATA).unwrap())?; + let input = fs::File::open(metadata_filename)?; let input = BufReader::new(input); let md: metadata::Metadata = match deserialize_from(input) { Ok(data) => data, @@ -946,7 +981,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); let app_parsing_start = SystemTime::now(); - let app_file = fs::File::open(&matches.value_of(APP).unwrap())?; + let app_file = fs::File::open(app_filename)?; let app_mmap = unsafe { Mmap::map(&app_file)? }; let app_data = &*app_mmap; let app_obj = match object::File::parse(app_data) { @@ -962,7 +997,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result { let exec_file = fs::OpenOptions::new() .read(true) .write(true) - .open(&matches.value_of(OUT).unwrap())?; + .open(out_filename)?; let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; exec_file.set_len(max_out_len)?;