Merge pull request #3928 from roc-lang/windows-surgical-dll

windows surgical dll

crates/compiler/build/src/link.rs

@@ -121,11 +121,20 @@ pub fn build_zig_host_native(
         .env("HOME", env_home);
 
     if let Some(shared_lib_path) = shared_lib_path {
+        // with LLVM, the builtins are already part of the roc app,
+        // but with the dev backend, they are missing. To minimize work,
+        // we link them as part of the host executable
+        let builtins_obj = if target.contains("windows") {
+            bitcode::get_builtins_windows_obj_path()
+        } else {
+            bitcode::get_builtins_host_obj_path()
+        };
+
         command.args(&[
             "build-exe",
             "-fPIE",
             shared_lib_path.to_str().unwrap(),
-            &bitcode::get_builtins_host_obj_path(),
+            &builtins_obj,
         ]);
     } else {
         command.args(&["build-obj", "-fPIC"]);
@@ -482,15 +491,29 @@ pub fn rebuild_host(
             host_input_path.with_file_name("host.bc")
         }
     } else {
-        host_input_path.with_file_name(if shared_lib_path.is_some() {
-            "dynhost"
+        let os = roc_target::OperatingSystem::from(target.operating_system);
+
+        if shared_lib_path.is_some() {
+            let extension = match os {
+                roc_target::OperatingSystem::Windows => "exe",
+                roc_target::OperatingSystem::Unix => "",
+                roc_target::OperatingSystem::Wasi => "",
+            };
+
+            host_input_path
+                .with_file_name("dynhost")
+                .with_extension(extension)
         } else {
-            match roc_target::OperatingSystem::from(target.operating_system) {
-                roc_target::OperatingSystem::Windows => "host.obj",
-                roc_target::OperatingSystem::Unix => "host.o",
-                roc_target::OperatingSystem::Wasi => "host.o",
+            let extension = match os {
+                roc_target::OperatingSystem::Windows => "obj",
+                roc_target::OperatingSystem::Unix => "o",
+                roc_target::OperatingSystem::Wasi => "o",
+            };
+
+            host_input_path
+                .with_file_name("host")
+                .with_extension(extension)
         }
-        })
     };
 
     let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());

crates/compiler/builtins/bitcode/build.zig

@@ -42,6 +42,7 @@ pub fn build(b: *Builder) void {
 
     // Generate Object Files
     generateObjectFile(b, mode, host_target, main_path, "object", "builtins-host");
+    generateObjectFile(b, mode, windows64_target, main_path, "windows-x86_64-object", "builtins-windows-x86_64");
     generateObjectFile(b, mode, wasm32_target, main_path, "wasm32-object", "builtins-wasm32");
 
     removeInstallSteps(b);

crates/compiler/builtins/bitcode/src/main.zig

@@ -253,7 +253,9 @@ test "" {
 
 // Export it as weak incase it is already linked in by something else.
 comptime {
+    if (builtin.target.os.tag != .windows) {
         @export(__muloti4, .{ .name = "__muloti4", .linkage = .Weak });
+    }
 }
 fn __muloti4(a: i128, b: i128, overflow: *c_int) callconv(.C) i128 {
     // @setRuntimeSafety(std.builtin.is_test);

crates/compiler/builtins/build.rs

@@ -60,6 +60,12 @@ fn main() {
 
     generate_object_file(&bitcode_path, "object", BUILTINS_HOST_FILE);
 
+    generate_object_file(
+        &bitcode_path,
+        "windows-x86_64-object",
+        "builtins-windows-x86_64.obj",
+    );
+
     generate_object_file(&bitcode_path, "wasm32-object", "builtins-wasm32.o");
 
     copy_zig_builtins_to_target_dir(&bitcode_path);

crates/compiler/builtins/src/bitcode.rs

@@ -14,6 +14,17 @@ pub fn get_builtins_host_obj_path() -> String {
         .expect("Failed to convert builtins_host_path to str")
 }
 
+pub fn get_builtins_windows_obj_path() -> String {
+    let builtins_host_path = get_lib_path()
+        .expect(LIB_DIR_ERROR)
+        .join("builtins-windows-x86_64.obj");
+
+    builtins_host_path
+        .into_os_string()
+        .into_string()
+        .expect("Failed to convert builtins_host_path to str")
+}
+
 pub fn get_builtins_wasm32_obj_path() -> String {
     let builtins_wasm32_path = get_lib_path()
         .expect(LIB_DIR_ERROR)

crates/linker/src/generate_dylib/elf64.rs

@@ -3,23 +3,10 @@ use object::read::elf::{FileHeader, ProgramHeader, SectionHeader, Sym};
 use object::Endianness;
 
 use object::write::elf::Writer;
-use target_lexicon::Triple;
-
-// an empty shared library, that we build on top of
-const DUMMY_ELF64: &[u8] = include_bytes!("../dummy-elf64-x86-64.so");
 
 // index of the dynamic section
 const DYMAMIC_SECTION: usize = 4;
 
-pub fn generate(target: &Triple, custom_names: &[String]) -> object::read::Result<Vec<u8>> {
-    match target.binary_format {
-        target_lexicon::BinaryFormat::Elf => create_dylib_elf64(DUMMY_ELF64, custom_names),
-        target_lexicon::BinaryFormat::Macho => todo!("macho dylib creation"),
-        target_lexicon::BinaryFormat::Coff => todo!("coff dylib creation"),
-        other => unimplemented!("dylib creation for {:?}", other),
-    }
-}
-
 #[derive(Debug)]
 struct Dynamic {
     tag: u32,
@@ -145,7 +132,10 @@ pub const fn round_up_to_alignment(width: usize, alignment: usize) -> usize {
     }
 }
 
-fn create_dylib_elf64(in_data: &[u8], custom_names: &[String]) -> object::read::Result<Vec<u8>> {
+pub fn create_dylib_elf64(
+    in_data: &[u8],
+    custom_names: &[String],
+) -> object::read::Result<Vec<u8>> {
     let in_elf: &elf::FileHeader64<Endianness> = elf::FileHeader64::parse(in_data)?;
     let endian = in_elf.endian()?;
     let in_segments = in_elf.program_headers(endian, in_data)?;

crates/linker/src/generate_dylib/mod.rs

@@ -0,0 +1,63 @@
+use target_lexicon::Triple;
+
+mod elf64;
+mod pe;
+
+/// an empty shared library, that we build on top of
+const DUMMY_ELF64: &[u8] = include_bytes!("../../dummy-elf64-x86-64.so");
+
+pub fn generate(target: &Triple, custom_names: &[String]) -> object::read::Result<Vec<u8>> {
+    match target.binary_format {
+        target_lexicon::BinaryFormat::Elf => elf64::create_dylib_elf64(DUMMY_ELF64, custom_names),
+        target_lexicon::BinaryFormat::Macho => todo!("macho dylib creation"),
+        target_lexicon::BinaryFormat::Coff => Ok(pe::synthetic_dll(custom_names)),
+        other => unimplemented!("dylib creation for {:?}", other),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use object::Object;
+
+    fn check_exports(target: &Triple) {
+        let custom_names = ["foo".to_string(), "bar".to_string()];
+
+        let bytes = generate(target, &custom_names).unwrap();
+        let object = object::File::parse(bytes.as_slice()).unwrap();
+
+        let exports = object.exports().unwrap();
+        for custom in custom_names {
+            assert!(
+                exports.iter().any(|e| e.name() == custom.as_bytes()),
+                "missing {}",
+                &custom
+            );
+        }
+    }
+
+    #[test]
+    fn check_exports_elf64() {
+        let target = target_lexicon::Triple {
+            architecture: target_lexicon::Architecture::X86_64,
+            operating_system: target_lexicon::OperatingSystem::Linux,
+            binary_format: target_lexicon::BinaryFormat::Elf,
+            ..target_lexicon::Triple::host()
+        };
+
+        check_exports(&target);
+    }
+
+    #[test]
+    fn check_exports_coff() {
+        let target = target_lexicon::Triple {
+            architecture: target_lexicon::Architecture::X86_64,
+            operating_system: target_lexicon::OperatingSystem::Windows,
+            binary_format: target_lexicon::BinaryFormat::Coff,
+            ..target_lexicon::Triple::host()
+        };
+
+        check_exports(&target);
+    }
+}

crates/linker/src/generate_dylib/pe.rs

@@ -0,0 +1,160 @@
+use object::pe;
+use object::LittleEndian as LE;
+
+fn synthetic_image_export_directory(
+    name: &str,
+    virtual_address: u32,
+    custom_names: &[String],
+) -> pe::ImageExportDirectory {
+    use object::{U16, U32};
+
+    let directory_size = std::mem::size_of::<pe::ImageExportDirectory>() as u32;
+
+    // actual data is after the directory and the name of the dll file (null-terminated)
+    let start = virtual_address + directory_size + name.len() as u32 + 1;
+
+    let address_of_functions = start;
+    let address_of_names = address_of_functions + 4 * custom_names.len() as u32;
+    let address_of_name_ordinals = address_of_names + 4 * custom_names.len() as u32;
+
+    pe::ImageExportDirectory {
+        characteristics: U32::new(LE, 0),
+        time_date_stamp: U32::new(LE, 0),
+        major_version: U16::new(LE, 0),
+        minor_version: U16::new(LE, 0),
+        name: U32::new(LE, virtual_address + directory_size),
+        base: U32::new(LE, 0),
+        number_of_functions: U32::new(LE, custom_names.len() as u32),
+        number_of_names: U32::new(LE, custom_names.len() as u32),
+        address_of_functions: U32::new(LE, address_of_functions),
+        address_of_names: U32::new(LE, address_of_names),
+        address_of_name_ordinals: U32::new(LE, address_of_name_ordinals),
+    }
+}
+
+fn synthetic_export_dir(virtual_address: u32, custom_names: &[String]) -> Vec<u8> {
+    let mut vec = vec![0; std::mem::size_of::<pe::ImageExportDirectory>()];
+
+    let ptr = vec.as_mut_ptr();
+
+    let name = "roc-cheaty-lib.dll";
+    let directory = synthetic_image_export_directory(name, virtual_address, custom_names);
+
+    unsafe {
+        std::ptr::write_unaligned(ptr as *mut pe::ImageExportDirectory, directory);
+    }
+
+    // write the .dll name, null-terminated
+    vec.extend(name.as_bytes());
+    vec.push(0);
+
+    let n = custom_names.len();
+
+    // Unsure what this one does; it does not seem important for our purposes
+    //
+    // Export Address Table -- Ordinal Base 0
+    //  [   1] +base[   1] 1020 Export RVA
+    //  [   2] +base[   2] d030 Export RVA
+    for _ in custom_names {
+        vec.extend(42u32.to_le_bytes());
+    }
+
+    // Maps the index to a name
+    //
+    // [Ordinal/Name Pointer] Table
+    //  [   1] _CRT_INIT
+    //  [   2] _CRT_MT
+    let mut acc = directory.address_of_name_ordinals.get(LE) + n as u32 * 2;
+    for name in custom_names {
+        vec.extend(acc.to_le_bytes());
+        acc += name.len() as u32 + 1;
+    }
+
+    // the ordinals, which just map to the index in our case
+    for (i, _) in custom_names.iter().enumerate() {
+        vec.extend((i as u16).to_le_bytes());
+    }
+
+    // write out the names of the symbols, as null-terminated strings
+    for name in custom_names {
+        vec.extend(name.as_bytes());
+        vec.push(0);
+    }
+
+    vec
+}
+
+pub fn synthetic_dll(custom_names: &[String]) -> Vec<u8> {
+    let mut out_data = Vec::new();
+    let mut writer = object::write::pe::Writer::new(true, 8, 8, &mut out_data);
+
+    // fairly randomly chosen. Not sure if this is relevant
+    let virtual_address = 0x138;
+
+    let exports = synthetic_export_dir(virtual_address, custom_names);
+
+    // Reserve file ranges and virtual addresses.
+    writer.reserve_dos_header_and_stub();
+
+    // we will have one header: the export directory
+    writer.reserve_nt_headers(1);
+
+    writer.set_data_directory(
+        pe::IMAGE_DIRECTORY_ENTRY_EXPORT,
+        virtual_address,
+        exports.len() as _,
+    );
+
+    writer.reserve_section_headers(1);
+
+    // we store the export directory in a .rdata section
+    let rdata_section: (_, Vec<u8>) = {
+        let range = writer.reserve_section(
+            *b".rdata\0\0",
+            1073741888,
+            // virtual size
+            exports.len() as u32,
+            // size_of_raw_data
+            exports.len() as u32,
+        );
+
+        (range.file_offset, exports)
+    };
+
+    // Start writing.
+    writer.write_dos_header_and_stub().unwrap();
+
+    // the header on my machine
+    let headers = object::write::pe::NtHeaders {
+        machine: 34404,
+        time_date_stamp: 1661696130,
+        characteristics: 8226,
+        major_linker_version: 14,
+        minor_linker_version: 0,
+        address_of_entry_point: 4560,
+        image_base: 6442450944,
+        major_operating_system_version: 6,
+        minor_operating_system_version: 0,
+        major_image_version: 0,
+        minor_image_version: 0,
+        major_subsystem_version: 6,
+        minor_subsystem_version: 0,
+        subsystem: 2,
+        dll_characteristics: 352,
+        size_of_stack_reserve: 1048576,
+        size_of_stack_commit: 4096,
+        size_of_heap_reserve: 1048576,
+        size_of_heap_commit: 4096,
+    };
+
+    writer.write_nt_headers(headers);
+
+    writer.write_section_headers();
+
+    let (offset, data) = rdata_section;
+    writer.write_section(offset, &data);
+
+    debug_assert_eq!(writer.reserved_len() as usize, writer.len());
+
+    out_data
+}

crates/linker/src/lib.rs

@@ -51,22 +51,35 @@ fn report_timing(label: &str, duration: Duration) {
 }
 
 pub fn supported(link_type: LinkType, target: &Triple) -> bool {
-    matches!(
-        (link_type, target),
-        (
-            LinkType::Executable,
+    if let LinkType::Executable = link_type {
+        match target {
             Triple {
                 architecture: target_lexicon::Architecture::X86_64,
                 operating_system: target_lexicon::OperatingSystem::Linux,
                 binary_format: target_lexicon::BinaryFormat::Elf,
                 ..
-            } // | Triple {
-              //     operating_system: target_lexicon::OperatingSystem::Darwin,
-              //     binary_format: target_lexicon::BinaryFormat::Macho,
-              //     ..
-              // }
-        )
-    )
+            } => true,
+
+            // macho support is incomplete
+            Triple {
+                operating_system: target_lexicon::OperatingSystem::Darwin,
+                binary_format: target_lexicon::BinaryFormat::Macho,
+                ..
+            } => false,
+
+            // windows support is incomplete
+            Triple {
+                architecture: target_lexicon::Architecture::X86_64,
+                operating_system: target_lexicon::OperatingSystem::Windows,
+                binary_format: target_lexicon::BinaryFormat::Coff,
+                ..
+            } => false,
+
+            _ => false,
+        }
+    } else {
+        false
+    }
 }
 
 pub fn build_and_preprocess_host(
@@ -77,7 +90,12 @@ pub fn build_and_preprocess_host(
     exposed_to_host: Vec<String>,
     exported_closure_types: Vec<String>,
 ) {
-    let dummy_lib = host_input_path.with_file_name("libapp.so");
+    let dummy_lib = if let target_lexicon::OperatingSystem::Windows = target.operating_system {
+        host_input_path.with_file_name("libapp.obj")
+    } else {
+        host_input_path.with_file_name("libapp.so")
+    };
+
     generate_dynamic_lib(target, exposed_to_host, exported_closure_types, &dummy_lib);
     rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib));
     let dynhost = host_input_path.with_file_name("dynhost");
@@ -136,11 +154,15 @@ fn generate_dynamic_lib(
         }
     }
 
+    // on windows (PE) binary search is used on the symbols,
+    // so they must be in alphabetical order
+    custom_names.sort_unstable();
+
     if !dummy_lib_is_up_to_date(target, dummy_lib_path, &custom_names) {
         let bytes = crate::generate_dylib::generate(target, &custom_names)
-            .unwrap_or_else(|e| internal_error!("{}", e));
+            .unwrap_or_else(|e| internal_error!("{e}"));
 
-        std::fs::write(dummy_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{}", e))
+        std::fs::write(dummy_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{e}"))
     }
 }
 
@@ -3016,7 +3038,7 @@ mod tests {
     }
 
     #[test]
-    fn collect_undefined_symbols() {
+    fn collect_undefined_symbols_elf() {
         let object = object::File::parse(ELF64_DYNHOST).unwrap();
 
         let mut triple = Triple::host();