Merge pull request #4380 from roc-lang/dummy-lib

Add a roc subcommand for generating the stub shared lib

Cargo.lock

@@ -3747,7 +3747,6 @@ version = "0.0.1"
 dependencies = [
  "bincode",
  "bumpalo",
- "clap 3.2.20",
  "iced-x86",
  "indoc",
  "libc",
@@ -3757,7 +3756,9 @@ dependencies = [
  "roc_build",
  "roc_collections",
  "roc_error_macros",
+ "roc_load",
  "roc_mono",
+ "roc_reporting",
  "serde",
  "target-lexicon",
  "tempfile",

crates/cli/src/lib.rs

@@ -44,6 +44,7 @@ pub const CMD_VERSION: &str = "version";
 pub const CMD_FORMAT: &str = "format";
 pub const CMD_TEST: &str = "test";
 pub const CMD_GLUE: &str = "glue";
+pub const CMD_GEN_STUB_LIB: &str = "gen-stub-lib";
 
 pub const FLAG_DEBUG: &str = "debug";
 pub const FLAG_DEV: &str = "dev";
@@ -276,6 +277,23 @@ pub fn build_app<'a>() -> Command<'a> {
                     .required(true)
             )
         )
+        .subcommand(Command::new(CMD_GEN_STUB_LIB)
+            .about("Generate a stubbed shared library that can be used for linking a platform binary.\nThe stubbed library has prototypes, but no function bodies.\n\nNote: This command will be removed in favor of just using `roc build` once all platforms support the surgical linker")
+            .arg(
+                Arg::new(ROC_FILE)
+                    .help("The .roc file for an app using the platform")
+                    .allow_invalid_utf8(true)
+                    .required(true)
+            )
+            .arg(
+                Arg::new(FLAG_TARGET)
+                    .long(FLAG_TARGET)
+                    .help("Choose a different target")
+                    .default_value(Target::default().as_str())
+                    .possible_values(Target::OPTIONS)
+                    .required(false),
+            )
+        )
         .trailing_var_arg(true)
         .arg(flag_optimize)
             .arg(flag_max_threads.clone())

crates/cli/src/main.rs

@@ -2,9 +2,9 @@ use roc_build::link::LinkType;
 use roc_cli::build::check_file;
 use roc_cli::{
     build_app, format, test, BuildConfig, FormatMode, Target, CMD_BUILD, CMD_CHECK, CMD_DEV,
-    CMD_DOCS, CMD_EDIT, CMD_FORMAT, CMD_GLUE, CMD_REPL, CMD_RUN, CMD_TEST, CMD_VERSION,
-    DIRECTORY_OR_FILES, FLAG_CHECK, FLAG_LIB, FLAG_NO_LINK, FLAG_TARGET, FLAG_TIME, GLUE_FILE,
-    ROC_FILE,
+    CMD_DOCS, CMD_EDIT, CMD_FORMAT, CMD_GEN_STUB_LIB, CMD_GLUE, CMD_REPL, CMD_RUN, CMD_TEST,
+    CMD_VERSION, DIRECTORY_OR_FILES, FLAG_CHECK, FLAG_LIB, FLAG_NO_LINK, FLAG_TARGET, FLAG_TIME,
+    GLUE_FILE, ROC_FILE,
 };
 use roc_docs::generate_docs_html;
 use roc_error_macros::user_error;
@@ -93,6 +93,12 @@ fn main() -> io::Result<()> {
                 Ok(1)
             }
         }
+        Some((CMD_GEN_STUB_LIB, matches)) => {
+            let input_path = Path::new(matches.value_of_os(ROC_FILE).unwrap());
+            let target: Target = matches.value_of_t(FLAG_TARGET).unwrap_or_default();
+
+            roc_linker::generate_stub_lib(input_path, &target.to_triple())
+        }
         Some((CMD_BUILD, matches)) => {
             let target: Target = matches.value_of_t(FLAG_TARGET).unwrap_or_default();
 

crates/linker/Cargo.toml

@@ -16,8 +16,9 @@ roc_mono = { path = "../compiler/mono" }
 roc_build = { path = "../compiler/build" }
 roc_collections = { path = "../compiler/collections" }
 roc_error_macros = { path = "../error_macros" }
+roc_load = { path = "../compiler/load" }
+roc_reporting = { path = "../reporting" }
 bumpalo = { version = "3.11.0", features = ["collections"] }
-clap = { version = "3.2.20", default-features = false, features = ["std", "color", "suggestions"] }
 iced-x86 = { version = "1.15.0", default-features = false, features = ["std", "decoder", "op_code_info", "instr_info"] }
 memmap2 = "0.5.7"
 object = { version = "0.29.0", features = ["read", "write"] }

crates/linker/src/generate_dylib/macho.rs

@@ -0,0 +1,94 @@
+use object::write;
+use object::{Architecture, BinaryFormat, Endianness, SymbolFlags, SymbolKind, SymbolScope};
+use roc_error_macros::internal_error;
+use std::path::Path;
+use std::process::Command;
+use target_lexicon::Triple;
+use tempfile::Builder;
+
+// TODO: Eventually do this from scratch and in memory instead of with ld.
+pub fn create_dylib_macho(
+    custom_names: &[String],
+    triple: &Triple,
+) -> object::read::Result<Vec<u8>> {
+    let dummy_obj_file = Builder::new()
+        .prefix("roc_lib")
+        .suffix(".o")
+        .tempfile()
+        .unwrap_or_else(|e| internal_error!("{}", e));
+    let dummy_obj_file = dummy_obj_file.path();
+    let tmp = tempfile::tempdir().unwrap_or_else(|e| internal_error!("{}", e));
+    let dummy_lib_file = tmp.path().to_path_buf().with_file_name("libapp.so");
+
+    let obj_target = BinaryFormat::MachO;
+    let obj_arch = match triple.architecture {
+        target_lexicon::Architecture::X86_64 => Architecture::X86_64,
+        target_lexicon::Architecture::Aarch64(_) => Architecture::Aarch64,
+        _ => {
+            // We should have verified this via supported() before calling this function
+            unreachable!()
+        }
+    };
+    let mut out_object = write::Object::new(obj_target, obj_arch, Endianness::Little);
+
+    let text_section = out_object.section_id(write::StandardSection::Text);
+
+    for name in custom_names {
+        out_object.add_symbol(write::Symbol {
+            name: name.as_bytes().to_vec(),
+            value: 0,
+            size: 0,
+            kind: SymbolKind::Text,
+            scope: SymbolScope::Dynamic,
+            weak: false,
+            section: write::SymbolSection::Section(text_section),
+            flags: SymbolFlags::None,
+        });
+    }
+
+    std::fs::write(
+        &dummy_obj_file,
+        out_object.write().expect("failed to build output object"),
+    )
+    .expect("failed to write object to file");
+
+    // This path only exists on macOS Big Sur, and it causes ld errors
+    // on Catalina if it's specified with -L, so we replace it with a
+    // redundant -lSystem if the directory isn't there.
+    let big_sur_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib";
+    let big_sur_fix = if Path::new(big_sur_path).exists() {
+        "-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"
+    } else {
+        "-lSystem" // We say -lSystem twice in the case of non-Big-Sur OSes, but it's fine.
+    };
+
+    let ld_flag_soname = "-install_name";
+    let ld_prefix_args = [big_sur_fix, "-lSystem", "-dylib"];
+
+    let output = Command::new("ld")
+        .args(ld_prefix_args)
+        .args(&[
+            ld_flag_soname,
+            dummy_lib_file.file_name().unwrap().to_str().unwrap(),
+            dummy_obj_file.to_str().unwrap(),
+            "-o",
+            dummy_lib_file.to_str().unwrap(),
+        ])
+        .output()
+        .unwrap();
+
+    if !output.status.success() {
+        match std::str::from_utf8(&output.stderr) {
+            Ok(stderr) => panic!(
+                "Failed to link dummy shared library - stderr of the `ld` command was:\n{}",
+                stderr
+            ),
+            Err(utf8_err) => panic!(
+                "Failed to link dummy shared library  - stderr of the `ld` command was invalid utf8 ({:?})",
+                utf8_err
+            ),
+        }
+    }
+
+    Ok(std::fs::read(dummy_lib_file).expect("Failed to load dummy library"))
+}

crates/linker/src/generate_dylib/mod.rs

@@ -1,6 +1,7 @@
 use target_lexicon::Triple;
 
 mod elf64;
+mod macho;
 mod pe;
 
 #[cfg(test)]
@@ -14,7 +15,7 @@ pub(crate) use pe::APP_DLL;
 pub fn generate(target: &Triple, custom_names: &[String]) -> object::read::Result<Vec<u8>> {
     match target.binary_format {
         target_lexicon::BinaryFormat::Elf => elf64::create_dylib_elf64(custom_names),
-        target_lexicon::BinaryFormat::Macho => todo!("macho dylib creation"),
+        target_lexicon::BinaryFormat::Macho => macho::create_dylib_macho(custom_names, target),
         target_lexicon::BinaryFormat::Coff => Ok(pe::synthetic_dll(custom_names)),
         other => unimplemented!("dylib creation for {:?}", other),
     }

crates/linker/src/lib.rs

@@ -2,7 +2,9 @@ use memmap2::{Mmap, MmapMut};
 use object::Object;
 use roc_build::link::{rebuild_host, LinkType};
 use roc_error_macros::internal_error;
+use roc_load::{EntryPoint, ExecutionMode, LoadConfig, Threading};
 use roc_mono::ir::OptLevel;
+use roc_reporting::report::RenderTarget;
 use std::cmp::Ordering;
 use std::mem;
 use std::path::{Path, PathBuf};
@@ -54,7 +56,7 @@ pub fn build_and_preprocess_host(
     exposed_to_host: Vec<String>,
     exported_closure_types: Vec<String>,
 ) {
-    let dummy_lib = if let target_lexicon::OperatingSystem::Windows = target.operating_system {
+    let stub_lib = if let target_lexicon::OperatingSystem::Windows = target.operating_system {
         host_input_path.with_file_name("libapp.dll")
     } else {
         host_input_path.with_file_name("libapp.so")
@@ -66,9 +68,9 @@ pub fn build_and_preprocess_host(
         host_input_path.with_file_name("dynhost")
     };
 
-    let dummy_dll_symbols = make_dummy_dll_symbols(exposed_to_host, exported_closure_types);
-    generate_dynamic_lib(target, &dummy_dll_symbols, &dummy_lib);
-    rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib));
+    let stub_dll_symbols = make_stub_dll_symbols(exposed_to_host, exported_closure_types);
+    generate_dynamic_lib(target, &stub_dll_symbols, &stub_lib);
+    rebuild_host(opt_level, target, host_input_path, Some(&stub_lib));
     let metadata = host_input_path.with_file_name("metadata");
     // let prehost = host_input_path.with_file_name("preprocessedhost");
 
@@ -77,8 +79,8 @@ pub fn build_and_preprocess_host(
         &dynhost,
         &metadata,
         preprocessed_host_path,
-        &dummy_lib,
-        &dummy_dll_symbols,
+        &stub_lib,
+        &stub_dll_symbols,
         false,
         false,
     )
@@ -94,7 +96,69 @@ pub fn link_preprocessed_host(
     surgery(roc_app_bytes, &metadata, binary_path, false, false, target)
 }
 
-fn make_dummy_dll_symbols(
+// Exposed function to load a platform file and generate a stub lib for it.
+pub fn generate_stub_lib(input_path: &Path, triple: &Triple) -> std::io::Result<i32> {
+    // Note: this should theoretically just be able to load the host, I think.
+    // Instead, I am loading an entire app because that was simpler and had example code.
+    // If this was expected to stay around for the the long term, we should change it.
+    // But hopefully it will be removable once we have surgical linking on all platforms.
+    let target_info = triple.into();
+    let arena = &bumpalo::Bump::new();
+    let subs_by_module = Default::default();
+    let loaded = roc_load::load_and_monomorphize(
+        arena,
+        input_path.to_path_buf(),
+        subs_by_module,
+        LoadConfig {
+            target_info,
+            render: RenderTarget::Generic,
+            threading: Threading::AllAvailable,
+            exec_mode: ExecutionMode::Executable,
+        },
+    )
+    .unwrap_or_else(|problem| todo!("{:?}", problem));
+
+    let exposed_to_host = loaded
+        .exposed_to_host
+        .values
+        .keys()
+        .map(|x| x.as_str(&loaded.interns).to_string())
+        .collect();
+
+    let exported_closure_types = loaded
+        .exposed_to_host
+        .closure_types
+        .iter()
+        .map(|x| {
+            format!(
+                "{}_{}",
+                x.module_string(&loaded.interns),
+                x.as_str(&loaded.interns)
+            )
+        })
+        .collect();
+
+    if let EntryPoint::Executable { platform_path, .. } = &loaded.entry_point {
+        let platform_path = input_path
+            .to_path_buf()
+            .parent()
+            .unwrap()
+            .join(platform_path);
+        let stub_lib = if let target_lexicon::OperatingSystem::Windows = triple.operating_system {
+            platform_path.with_file_name("libapp.obj")
+        } else {
+            platform_path.with_file_name("libapp.so")
+        };
+
+        let stub_dll_symbols = make_stub_dll_symbols(exposed_to_host, exported_closure_types);
+        generate_dynamic_lib(triple, &stub_dll_symbols, &stub_lib);
+    } else {
+        unreachable!();
+    };
+    Ok(0)
+}
+
+fn make_stub_dll_symbols(
     exposed_to_host: Vec<String>,
     exported_closure_types: Vec<String>,
 ) -> Vec<String> {
@@ -123,23 +187,23 @@ fn make_dummy_dll_symbols(
     custom_names
 }
 
-fn generate_dynamic_lib(target: &Triple, dummy_dll_symbols: &[String], dummy_lib_path: &Path) {
-    if !dummy_lib_is_up_to_date(target, dummy_lib_path, dummy_dll_symbols) {
-        let bytes = crate::generate_dylib::generate(target, dummy_dll_symbols)
+fn generate_dynamic_lib(target: &Triple, stub_dll_symbols: &[String], stub_lib_path: &Path) {
+    if !stub_lib_is_up_to_date(target, stub_lib_path, stub_dll_symbols) {
+        let bytes = crate::generate_dylib::generate(target, stub_dll_symbols)
             .unwrap_or_else(|e| internal_error!("{e}"));
 
-        std::fs::write(dummy_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{e}"));
+        std::fs::write(stub_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{e}"));
 
         if let target_lexicon::OperatingSystem::Windows = target.operating_system {
-            generate_import_library(dummy_lib_path, dummy_dll_symbols);
+            generate_import_library(stub_lib_path, stub_dll_symbols);
         }
     }
 }
 
-fn generate_import_library(dummy_lib_path: &Path, custom_names: &[String]) {
+fn generate_import_library(stub_lib_path: &Path, custom_names: &[String]) {
     let def_file_content = generate_def_file(custom_names).expect("write to string never fails");
 
-    let mut def_path = dummy_lib_path.to_owned();
+    let mut def_path = stub_lib_path.to_owned();
     def_path.set_extension("def");
 
     std::fs::write(def_path, def_file_content.as_bytes())
@@ -161,7 +225,7 @@ fn generate_import_library(dummy_lib_path: &Path, custom_names: &[String]) {
     //
     // > https://github.com/messense/implib-rs
     let output = std::process::Command::new(&zig)
-        .current_dir(dummy_lib_path.parent().unwrap())
+        .current_dir(stub_lib_path.parent().unwrap())
         .args(&[
             "dlltool",
             "-d",
@@ -226,20 +290,16 @@ fn object_matches_target<'a>(target: &Triple, object: &object::File<'a, &'a [u8]
     }
 }
 
-/// Checks whether the dummy `.dll/.so` is up to date, in other words that it exports exactly the
+/// Checks whether the stub `.dll/.so` is up to date, in other words that it exports exactly the
 /// symbols that it is supposed to export, and is built for the right target. If this is the case,
-/// we can skip rebuildingthe dummy lib.
-fn dummy_lib_is_up_to_date(
-    target: &Triple,
-    dummy_lib_path: &Path,
-    custom_names: &[String],
-) -> bool {
-    if !std::path::Path::exists(dummy_lib_path) {
+/// we can skip rebuildingthe stub lib.
+fn stub_lib_is_up_to_date(target: &Triple, stub_lib_path: &Path, custom_names: &[String]) -> bool {
+    if !std::path::Path::exists(stub_lib_path) {
         return false;
     }
 
-    let dummy_lib = open_mmap(dummy_lib_path);
-    let object = object::File::parse(&*dummy_lib).unwrap();
+    let stub_lib = open_mmap(stub_lib_path);
+    let object = object::File::parse(&*stub_lib).unwrap();
 
     // the user may have been cross-compiling.
     // The dynhost on disk must match our current target
@@ -265,7 +325,7 @@ fn preprocess(
     metadata_path: &Path,
     preprocessed_path: &Path,
     shared_lib: &Path,
-    dummy_dll_symbols: &[String],
+    stub_dll_symbols: &[String],
     verbose: bool,
     time: bool,
 ) {
@@ -307,7 +367,7 @@ fn preprocess(
                 host_exe_path,
                 metadata_path,
                 preprocessed_path,
-                dummy_dll_symbols,
+                stub_dll_symbols,
                 verbose,
                 time,
             )