mirror of
https://github.com/roc-lang/roc.git
synced 2025-08-03 11:52:19 +00:00
Merge remote-tracking branch 'origin/main' into roc-dev-inline-expects
This commit is contained in:
Folkert
commit
dbd0d30893
98 changed files with 4649 additions and 1433 deletions
|
|
@ -16,8 +16,9 @@ roc_mono = { path = "../compiler/mono" }
|
|||
roc_build = { path = "../compiler/build" }
|
||||
roc_collections = { path = "../compiler/collections" }
|
||||
roc_error_macros = { path = "../error_macros" }
|
||||
roc_load = { path = "../compiler/load" }
|
||||
roc_reporting = { path = "../reporting" }
|
||||
bumpalo = { version = "3.11.0", features = ["collections"] }
|
||||
clap = { version = "3.2.20", default-features = false, features = ["std", "color", "suggestions"] }
|
||||
iced-x86 = { version = "1.15.0", default-features = false, features = ["std", "decoder", "op_code_info", "instr_info"] }
|
||||
memmap2 = "0.5.7"
|
||||
object = { version = "0.29.0", features = ["read", "write"] }
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -1644,7 +1644,7 @@ mod tests {
|
|||
let dylib_bytes = crate::generate_dylib::create_dylib_elf64(&names).unwrap();
|
||||
std::fs::write(dir.join("libapp.so"), dylib_bytes).unwrap();
|
||||
|
||||
// now we can compile the host (it uses libapp.obj, hence the order here)
|
||||
// now we can compile the host (it uses libapp.so, hence the order here)
|
||||
let output = std::process::Command::new(&zig)
|
||||
.current_dir(dir)
|
||||
.args(&[
|
||||
|
|
|
|||
94
crates/linker/src/generate_dylib/macho.rs
Normal file
94
crates/linker/src/generate_dylib/macho.rs
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
use object::write;
|
||||
use object::{Architecture, BinaryFormat, Endianness, SymbolFlags, SymbolKind, SymbolScope};
|
||||
use roc_error_macros::internal_error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use target_lexicon::Triple;
|
||||
use tempfile::Builder;
|
||||
|
||||
// TODO: Eventually do this from scratch and in memory instead of with ld.
|
||||
pub fn create_dylib_macho(
|
||||
custom_names: &[String],
|
||||
triple: &Triple,
|
||||
) -> object::read::Result<Vec<u8>> {
|
||||
let dummy_obj_file = Builder::new()
|
||||
.prefix("roc_lib")
|
||||
.suffix(".o")
|
||||
.tempfile()
|
||||
.unwrap_or_else(|e| internal_error!("{}", e));
|
||||
let dummy_obj_file = dummy_obj_file.path();
|
||||
let tmp = tempfile::tempdir().unwrap_or_else(|e| internal_error!("{}", e));
|
||||
let dummy_lib_file = tmp.path().to_path_buf().with_file_name("libapp.so");
|
||||
|
||||
let obj_target = BinaryFormat::MachO;
|
||||
let obj_arch = match triple.architecture {
|
||||
target_lexicon::Architecture::X86_64 => Architecture::X86_64,
|
||||
target_lexicon::Architecture::Aarch64(_) => Architecture::Aarch64,
|
||||
_ => {
|
||||
// We should have verified this via supported() before calling this function
|
||||
unreachable!()
|
||||
}
|
||||
};
|
||||
let mut out_object = write::Object::new(obj_target, obj_arch, Endianness::Little);
|
||||
|
||||
let text_section = out_object.section_id(write::StandardSection::Text);
|
||||
|
||||
for name in custom_names {
|
||||
out_object.add_symbol(write::Symbol {
|
||||
name: name.as_bytes().to_vec(),
|
||||
value: 0,
|
||||
size: 0,
|
||||
kind: SymbolKind::Text,
|
||||
scope: SymbolScope::Dynamic,
|
||||
weak: false,
|
||||
section: write::SymbolSection::Section(text_section),
|
||||
flags: SymbolFlags::None,
|
||||
});
|
||||
}
|
||||
|
||||
std::fs::write(
|
||||
&dummy_obj_file,
|
||||
out_object.write().expect("failed to build output object"),
|
||||
)
|
||||
.expect("failed to write object to file");
|
||||
|
||||
// This path only exists on macOS Big Sur, and it causes ld errors
|
||||
// on Catalina if it's specified with -L, so we replace it with a
|
||||
// redundant -lSystem if the directory isn't there.
|
||||
let big_sur_path = "/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib";
|
||||
let big_sur_fix = if Path::new(big_sur_path).exists() {
|
||||
"-L/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib"
|
||||
} else {
|
||||
"-lSystem" // We say -lSystem twice in the case of non-Big-Sur OSes, but it's fine.
|
||||
};
|
||||
|
||||
let ld_flag_soname = "-install_name";
|
||||
let ld_prefix_args = [big_sur_fix, "-lSystem", "-dylib"];
|
||||
|
||||
let output = Command::new("ld")
|
||||
.args(ld_prefix_args)
|
||||
.args(&[
|
||||
ld_flag_soname,
|
||||
dummy_lib_file.file_name().unwrap().to_str().unwrap(),
|
||||
dummy_obj_file.to_str().unwrap(),
|
||||
"-o",
|
||||
dummy_lib_file.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
if !output.status.success() {
|
||||
match std::str::from_utf8(&output.stderr) {
|
||||
Ok(stderr) => panic!(
|
||||
"Failed to link dummy shared library - stderr of the `ld` command was:\n{}",
|
||||
stderr
|
||||
),
|
||||
Err(utf8_err) => panic!(
|
||||
"Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})",
|
||||
utf8_err
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(std::fs::read(dummy_lib_file).expect("Failed to load dummy library"))
|
||||
}
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
use target_lexicon::Triple;
|
||||
|
||||
mod elf64;
|
||||
mod macho;
|
||||
mod pe;
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -14,7 +15,7 @@ pub(crate) use pe::APP_DLL;
|
|||
pub fn generate(target: &Triple, custom_names: &[String]) -> object::read::Result<Vec<u8>> {
|
||||
match target.binary_format {
|
||||
target_lexicon::BinaryFormat::Elf => elf64::create_dylib_elf64(custom_names),
|
||||
target_lexicon::BinaryFormat::Macho => todo!("macho dylib creation"),
|
||||
target_lexicon::BinaryFormat::Macho => macho::create_dylib_macho(custom_names, target),
|
||||
target_lexicon::BinaryFormat::Coff => Ok(pe::synthetic_dll(custom_names)),
|
||||
other => unimplemented!("dylib creation for {:?}", other),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use object::pe;
|
||||
use object::LittleEndian as LE;
|
||||
|
||||
pub(crate) const APP_DLL: &str = "roc-cheaty-lib.dll";
|
||||
pub(crate) const APP_DLL: &str = "libapp.dll";
|
||||
|
||||
fn synthetic_image_export_directory(
|
||||
name: &str,
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ use memmap2::{Mmap, MmapMut};
|
|||
use object::Object;
|
||||
use roc_build::link::{rebuild_host, LinkType};
|
||||
use roc_error_macros::internal_error;
|
||||
use roc_load::{EntryPoint, ExecutionMode, LoadConfig, Threading};
|
||||
use roc_mono::ir::OptLevel;
|
||||
use roc_reporting::report::RenderTarget;
|
||||
use std::cmp::Ordering;
|
||||
use std::mem;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use target_lexicon::Triple;
|
||||
|
||||
mod elf;
|
||||
|
|
@ -54,8 +56,8 @@ pub fn build_and_preprocess_host(
|
|||
exposed_to_host: Vec<String>,
|
||||
exported_closure_types: Vec<String>,
|
||||
) {
|
||||
let dummy_lib = if let target_lexicon::OperatingSystem::Windows = target.operating_system {
|
||||
host_input_path.with_file_name("libapp.obj")
|
||||
let stub_lib = if let target_lexicon::OperatingSystem::Windows = target.operating_system {
|
||||
host_input_path.with_file_name("libapp.dll")
|
||||
} else {
|
||||
host_input_path.with_file_name("libapp.so")
|
||||
};
|
||||
|
|
@ -66,9 +68,9 @@ pub fn build_and_preprocess_host(
|
|||
host_input_path.with_file_name("dynhost")
|
||||
};
|
||||
|
||||
let dummy_dll_symbols = make_dummy_dll_symbols(exposed_to_host, exported_closure_types);
|
||||
generate_dynamic_lib(target, &dummy_dll_symbols, &dummy_lib);
|
||||
rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib));
|
||||
let stub_dll_symbols = make_stub_dll_symbols(exposed_to_host, exported_closure_types);
|
||||
generate_dynamic_lib(target, &stub_dll_symbols, &stub_lib);
|
||||
rebuild_host(opt_level, target, host_input_path, Some(&stub_lib));
|
||||
let metadata = host_input_path.with_file_name("metadata");
|
||||
// let prehost = host_input_path.with_file_name("preprocessedhost");
|
||||
|
||||
|
|
@ -77,8 +79,8 @@ pub fn build_and_preprocess_host(
|
|||
&dynhost,
|
||||
&metadata,
|
||||
preprocessed_host_path,
|
||||
&dummy_lib,
|
||||
&dummy_dll_symbols,
|
||||
&stub_lib,
|
||||
&stub_dll_symbols,
|
||||
false,
|
||||
false,
|
||||
)
|
||||
|
|
@ -94,7 +96,69 @@ pub fn link_preprocessed_host(
|
|||
surgery(roc_app_bytes, &metadata, binary_path, false, false, target)
|
||||
}
|
||||
|
||||
fn make_dummy_dll_symbols(
|
||||
// Exposed function to load a platform file and generate a stub lib for it.
|
||||
pub fn generate_stub_lib(input_path: &Path, triple: &Triple) -> std::io::Result<i32> {
|
||||
// Note: this should theoretically just be able to load the host, I think.
|
||||
// Instead, I am loading an entire app because that was simpler and had example code.
|
||||
// If this was expected to stay around for the the long term, we should change it.
|
||||
// But hopefully it will be removable once we have surgical linking on all platforms.
|
||||
let target_info = triple.into();
|
||||
let arena = &bumpalo::Bump::new();
|
||||
let subs_by_module = Default::default();
|
||||
let loaded = roc_load::load_and_monomorphize(
|
||||
arena,
|
||||
input_path.to_path_buf(),
|
||||
subs_by_module,
|
||||
LoadConfig {
|
||||
target_info,
|
||||
render: RenderTarget::Generic,
|
||||
threading: Threading::AllAvailable,
|
||||
exec_mode: ExecutionMode::Executable,
|
||||
},
|
||||
)
|
||||
.unwrap_or_else(|problem| todo!("{:?}", problem));
|
||||
|
||||
let exposed_to_host = loaded
|
||||
.exposed_to_host
|
||||
.values
|
||||
.keys()
|
||||
.map(|x| x.as_str(&loaded.interns).to_string())
|
||||
.collect();
|
||||
|
||||
let exported_closure_types = loaded
|
||||
.exposed_to_host
|
||||
.closure_types
|
||||
.iter()
|
||||
.map(|x| {
|
||||
format!(
|
||||
"{}_{}",
|
||||
x.module_string(&loaded.interns),
|
||||
x.as_str(&loaded.interns)
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if let EntryPoint::Executable { platform_path, .. } = &loaded.entry_point {
|
||||
let platform_path = input_path
|
||||
.to_path_buf()
|
||||
.parent()
|
||||
.unwrap()
|
||||
.join(platform_path);
|
||||
let stub_lib = if let target_lexicon::OperatingSystem::Windows = triple.operating_system {
|
||||
platform_path.with_file_name("libapp.obj")
|
||||
} else {
|
||||
platform_path.with_file_name("libapp.so")
|
||||
};
|
||||
|
||||
let stub_dll_symbols = make_stub_dll_symbols(exposed_to_host, exported_closure_types);
|
||||
generate_dynamic_lib(triple, &stub_dll_symbols, &stub_lib);
|
||||
} else {
|
||||
unreachable!();
|
||||
};
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
fn make_stub_dll_symbols(
|
||||
exposed_to_host: Vec<String>,
|
||||
exported_closure_types: Vec<String>,
|
||||
) -> Vec<String> {
|
||||
|
|
@ -123,15 +187,87 @@ fn make_dummy_dll_symbols(
|
|||
custom_names
|
||||
}
|
||||
|
||||
fn generate_dynamic_lib(target: &Triple, custom_names: &[String], dummy_lib_path: &Path) {
|
||||
if !dummy_lib_is_up_to_date(target, dummy_lib_path, custom_names) {
|
||||
let bytes = crate::generate_dylib::generate(target, custom_names)
|
||||
fn generate_dynamic_lib(target: &Triple, stub_dll_symbols: &[String], stub_lib_path: &Path) {
|
||||
if !stub_lib_is_up_to_date(target, stub_lib_path, stub_dll_symbols) {
|
||||
let bytes = crate::generate_dylib::generate(target, stub_dll_symbols)
|
||||
.unwrap_or_else(|e| internal_error!("{e}"));
|
||||
|
||||
std::fs::write(dummy_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{e}"))
|
||||
std::fs::write(stub_lib_path, &bytes).unwrap_or_else(|e| internal_error!("{e}"));
|
||||
|
||||
if let target_lexicon::OperatingSystem::Windows = target.operating_system {
|
||||
generate_import_library(stub_lib_path, stub_dll_symbols);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_import_library(stub_lib_path: &Path, custom_names: &[String]) {
|
||||
let def_file_content = generate_def_file(custom_names).expect("write to string never fails");
|
||||
|
||||
let mut def_path = stub_lib_path.to_owned();
|
||||
def_path.set_extension("def");
|
||||
|
||||
std::fs::write(def_path, def_file_content.as_bytes())
|
||||
.unwrap_or_else(|e| internal_error!("{e}"));
|
||||
|
||||
let mut def_filename = PathBuf::from(generate_dylib::APP_DLL);
|
||||
def_filename.set_extension("def");
|
||||
|
||||
let mut lib_filename = PathBuf::from(generate_dylib::APP_DLL);
|
||||
lib_filename.set_extension("lib");
|
||||
|
||||
let zig = std::env::var("ROC_ZIG").unwrap_or_else(|_| "zig".into());
|
||||
|
||||
// use zig to generate the .lib file. Here is a good description of what is in an import library
|
||||
//
|
||||
// > https://www.codeproject.com/Articles/1253835/The-Structure-of-import-Library-File-lib
|
||||
//
|
||||
// For when we want to do this in-memory in the future. We can also consider using
|
||||
//
|
||||
// > https://github.com/messense/implib-rs
|
||||
let output = std::process::Command::new(&zig)
|
||||
.current_dir(stub_lib_path.parent().unwrap())
|
||||
.args(&[
|
||||
"dlltool",
|
||||
"-d",
|
||||
def_filename.to_str().unwrap(),
|
||||
"-m",
|
||||
"i386:x86-64",
|
||||
"-D",
|
||||
generate_dylib::APP_DLL,
|
||||
"-l",
|
||||
lib_filename.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
if !output.status.success() {
|
||||
use std::io::Write;
|
||||
|
||||
std::io::stdout().write_all(&output.stdout).unwrap();
|
||||
std::io::stderr().write_all(&output.stderr).unwrap();
|
||||
|
||||
panic!("zig dlltool failed");
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_def_file(custom_names: &[String]) -> Result<String, std::fmt::Error> {
|
||||
use std::fmt::Write;
|
||||
|
||||
let mut def_file = String::new();
|
||||
|
||||
writeln!(def_file, "LIBRARY libapp")?;
|
||||
writeln!(def_file, "EXPORTS")?;
|
||||
|
||||
for (i, name) in custom_names.iter().enumerate() {
|
||||
// 1-indexed of course...
|
||||
let index = i + 1;
|
||||
|
||||
writeln!(def_file, " {name} @{index}")?;
|
||||
}
|
||||
|
||||
Ok(def_file)
|
||||
}
|
||||
|
||||
fn object_matches_target<'a>(target: &Triple, object: &object::File<'a, &'a [u8]>) -> bool {
|
||||
use target_lexicon::{Architecture as TLA, OperatingSystem as TLO};
|
||||
|
||||
|
|
@ -154,20 +290,16 @@ fn object_matches_target<'a>(target: &Triple, object: &object::File<'a, &'a [u8]
|
|||
}
|
||||
}
|
||||
|
||||
/// Checks whether the dummy `.dll/.so` is up to date, in other words that it exports exactly the
|
||||
/// Checks whether the stub `.dll/.so` is up to date, in other words that it exports exactly the
|
||||
/// symbols that it is supposed to export, and is built for the right target. If this is the case,
|
||||
/// we can skip rebuildingthe dummy lib.
|
||||
fn dummy_lib_is_up_to_date(
|
||||
target: &Triple,
|
||||
dummy_lib_path: &Path,
|
||||
custom_names: &[String],
|
||||
) -> bool {
|
||||
if !std::path::Path::exists(dummy_lib_path) {
|
||||
/// we can skip rebuildingthe stub lib.
|
||||
fn stub_lib_is_up_to_date(target: &Triple, stub_lib_path: &Path, custom_names: &[String]) -> bool {
|
||||
if !std::path::Path::exists(stub_lib_path) {
|
||||
return false;
|
||||
}
|
||||
|
||||
let dummy_lib = open_mmap(dummy_lib_path);
|
||||
let object = object::File::parse(&*dummy_lib).unwrap();
|
||||
let stub_lib = open_mmap(stub_lib_path);
|
||||
let object = object::File::parse(&*stub_lib).unwrap();
|
||||
|
||||
// the user may have been cross-compiling.
|
||||
// The dynhost on disk must match our current target
|
||||
|
|
@ -193,7 +325,7 @@ fn preprocess(
|
|||
metadata_path: &Path,
|
||||
preprocessed_path: &Path,
|
||||
shared_lib: &Path,
|
||||
dummy_dll_symbols: &[String],
|
||||
stub_dll_symbols: &[String],
|
||||
verbose: bool,
|
||||
time: bool,
|
||||
) {
|
||||
|
|
@ -235,7 +367,7 @@ fn preprocess(
|
|||
host_exe_path,
|
||||
metadata_path,
|
||||
preprocessed_path,
|
||||
dummy_dll_symbols,
|
||||
stub_dll_symbols,
|
||||
verbose,
|
||||
time,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ struct PeMetadata {
|
|||
thunks_start_offset_in_section: usize,
|
||||
|
||||
/// Virtual address of the .rdata section
|
||||
rdata_virtual_address: u32,
|
||||
dummy_dll_thunk_section_virtual_address: u32,
|
||||
|
||||
/// The offset into the file of the .reloc section
|
||||
reloc_offset_in_file: usize,
|
||||
|
|
@ -108,18 +108,16 @@ impl PeMetadata {
|
|||
.unwrap();
|
||||
|
||||
let dynamic_relocations = DynamicRelocationsPe::new(preprocessed_data);
|
||||
let thunks_start_offset_in_file =
|
||||
find_thunks_start_offset(preprocessed_data, &dynamic_relocations);
|
||||
|
||||
let rdata_section = dynhost_obj
|
||||
.sections()
|
||||
.find(|s| s.name() == Ok(".rdata"))
|
||||
.unwrap();
|
||||
let dummy_dll_thunks = find_thunks_start_offset(preprocessed_data, &dynamic_relocations);
|
||||
let thunks_start_offset_in_file = dummy_dll_thunks.offset_in_file as usize;
|
||||
|
||||
let thunks_start_offset_in_section =
|
||||
thunks_start_offset_in_file - rdata_section.file_range().unwrap().0 as usize;
|
||||
let dummy_dll_thunk_section_virtual_address =
|
||||
dummy_dll_thunks.section.virtual_address.get(LE);
|
||||
|
||||
let rdata_virtual_address = rdata_section.address() as u32;
|
||||
let thunks_start_offset_in_section = (dummy_dll_thunks.offset_in_file
|
||||
- dummy_dll_thunks.section.pointer_to_raw_data.get(LE) as u64)
|
||||
as usize;
|
||||
|
||||
let (reloc_section_index, reloc_section) = dynhost_obj
|
||||
.sections()
|
||||
|
|
@ -175,7 +173,7 @@ impl PeMetadata {
|
|||
dynamic_relocations,
|
||||
thunks_start_offset_in_file,
|
||||
thunks_start_offset_in_section,
|
||||
rdata_virtual_address,
|
||||
dummy_dll_thunk_section_virtual_address,
|
||||
reloc_offset_in_file,
|
||||
reloc_section_index,
|
||||
}
|
||||
|
|
@ -191,6 +189,7 @@ pub(crate) fn preprocess_windows(
|
|||
_time: bool,
|
||||
) -> object::read::Result<()> {
|
||||
let data = open_mmap(host_exe_filename);
|
||||
|
||||
let new_sections = [*b".text\0\0\0", *b".rdata\0\0"];
|
||||
let mut preprocessed = Preprocessor::preprocess(
|
||||
preprocessed_filename,
|
||||
|
|
@ -214,23 +213,42 @@ pub(crate) fn preprocess_windows(
|
|||
dir.size.set(LE, new);
|
||||
}
|
||||
|
||||
// clear out the import table entry. we do implicitly assume that our dummy .dll is the last
|
||||
{
|
||||
const W: usize = std::mem::size_of::<ImageImportDescriptor>();
|
||||
|
||||
let start = md.dynamic_relocations.imports_offset_in_file as usize
|
||||
+ W * md.dynamic_relocations.dummy_import_index as usize;
|
||||
|
||||
for b in preprocessed[start..][..W].iter_mut() {
|
||||
*b = 0;
|
||||
}
|
||||
}
|
||||
remove_dummy_dll_import_table_entry(&mut preprocessed, &md);
|
||||
|
||||
md.write_to_file(metadata_filename);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_dummy_dll_import_table_entry(executable: &mut [u8], md: &PeMetadata) {
|
||||
const W: usize = std::mem::size_of::<ImageImportDescriptor>();
|
||||
|
||||
let dr = &md.dynamic_relocations;
|
||||
|
||||
// there is one zeroed-out descriptor at the back
|
||||
let count = dr.import_directory_size as usize / W - 1;
|
||||
|
||||
let descriptors = load_structs_inplace_mut::<ImageImportDescriptor>(
|
||||
executable,
|
||||
dr.import_directory_offset_in_file as usize,
|
||||
count,
|
||||
);
|
||||
|
||||
// move the dummy to the final position
|
||||
descriptors.swap(dr.dummy_import_index as usize, count - 1);
|
||||
|
||||
// make this the new zeroed-out descriptor
|
||||
if let Some(d) = descriptors.last_mut() {
|
||||
*d = ImageImportDescriptor {
|
||||
original_first_thunk: Default::default(),
|
||||
time_date_stamp: Default::default(),
|
||||
forwarder_chain: Default::default(),
|
||||
name: Default::default(),
|
||||
first_thunk: Default::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_bytes: &[u8]) {
|
||||
let md = PeMetadata::read_from_file(metadata_path);
|
||||
|
||||
|
|
@ -256,7 +274,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
) as u64;
|
||||
|
||||
let mut section_file_offset = md.dynhost_file_size;
|
||||
let mut virtual_address = (app_code_section_va - image_base) as u32;
|
||||
let mut section_virtual_address = (app_code_section_va - image_base) as u32;
|
||||
|
||||
// find the location to write the section headers for our new sections
|
||||
let mut section_header_start = md.dynamic_relocations.section_headers_offset_in_file as usize
|
||||
|
|
@ -288,7 +306,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
// offset_in_section now becomes a proper virtual address
|
||||
for symbol in symbols.iter_mut() {
|
||||
if symbol.section_kind == kind {
|
||||
symbol.offset_in_section += image_base as usize + virtual_address as usize;
|
||||
symbol.offset_in_section += image_base as usize + section_virtual_address as usize;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -307,7 +325,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
section_header_start,
|
||||
section_file_offset,
|
||||
virtual_size,
|
||||
virtual_address,
|
||||
section_virtual_address,
|
||||
size_of_raw_data,
|
||||
);
|
||||
}
|
||||
|
|
@ -321,7 +339,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
section_header_start,
|
||||
section_file_offset,
|
||||
virtual_size,
|
||||
virtual_address,
|
||||
section_virtual_address,
|
||||
size_of_raw_data,
|
||||
);
|
||||
}
|
||||
|
|
@ -351,9 +369,10 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
// we implicitly only do 32-bit relocations
|
||||
debug_assert_eq!(relocation.size(), 32);
|
||||
|
||||
let delta =
|
||||
destination - virtual_address as i64 - *offset_in_section as i64
|
||||
+ relocation.addend();
|
||||
let delta = destination
|
||||
- section_virtual_address as i64
|
||||
- *offset_in_section as i64
|
||||
+ relocation.addend();
|
||||
|
||||
executable[offset + *offset_in_section as usize..][..4]
|
||||
.copy_from_slice(&(delta as i32).to_le_bytes());
|
||||
|
|
@ -364,8 +383,9 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
// we implicitly only do 32-bit relocations
|
||||
debug_assert_eq!(relocation.size(), 32);
|
||||
|
||||
let delta = destination - virtual_address as i64 - *offset_in_section as i64
|
||||
+ relocation.addend();
|
||||
let delta =
|
||||
destination - section_virtual_address as i64 - *offset_in_section as i64
|
||||
+ relocation.addend();
|
||||
|
||||
executable[offset + *offset_in_section as usize..][..4]
|
||||
.copy_from_slice(&(delta as i32).to_le_bytes());
|
||||
|
|
@ -391,7 +411,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
|
|||
|
||||
section_header_start += std::mem::size_of::<ImageSectionHeader>();
|
||||
section_file_offset += size_of_raw_data as usize;
|
||||
virtual_address += next_multiple_of(length, section_alignment) as u32;
|
||||
section_virtual_address += next_multiple_of(length, section_alignment) as u32;
|
||||
file_bytes_added += next_multiple_of(length, section_alignment) as u32;
|
||||
}
|
||||
|
||||
|
|
@ -431,7 +451,10 @@ struct DynamicRelocationsPe {
|
|||
section_offset_in_file: u32,
|
||||
|
||||
/// Offset in the file of the imports directory
|
||||
imports_offset_in_file: u32,
|
||||
import_directory_offset_in_file: u32,
|
||||
|
||||
/// Size in the file of the imports directory
|
||||
import_directory_size: u32,
|
||||
|
||||
/// Offset in the file of the data directories
|
||||
data_directories_offset_in_file: u32,
|
||||
|
|
@ -536,8 +559,9 @@ impl DynamicRelocationsPe {
|
|||
|
||||
let import_table = ImportTable::new(section_data, section_va, import_va);
|
||||
|
||||
let imports_offset_in_section = import_va.wrapping_sub(section_va);
|
||||
let imports_offset_in_file = offset_in_file + imports_offset_in_section;
|
||||
let (import_directory_offset_in_file, import_directory_size) = data_dir
|
||||
.file_range(§ions)
|
||||
.expect("import directory exists");
|
||||
|
||||
let (descriptor, dummy_import_index) = Self::find_roc_dummy_dll(&import_table)?.unwrap();
|
||||
|
||||
|
|
@ -546,10 +570,11 @@ impl DynamicRelocationsPe {
|
|||
address_and_offset: Default::default(),
|
||||
section_virtual_address: section_va,
|
||||
section_offset_in_file: offset_in_file,
|
||||
imports_offset_in_file,
|
||||
import_directory_offset_in_file,
|
||||
data_directories_offset_in_file,
|
||||
dummy_import_index,
|
||||
section_headers_offset_in_file,
|
||||
import_directory_size,
|
||||
};
|
||||
|
||||
this.append_roc_imports(&import_table, &descriptor)?;
|
||||
|
|
@ -820,12 +845,16 @@ impl Preprocessor {
|
|||
}
|
||||
}
|
||||
}
|
||||
struct DummyDllThunks<'a> {
|
||||
section: &'a object::pe::ImageSectionHeader,
|
||||
offset_in_file: u64,
|
||||
}
|
||||
|
||||
/// Find the place in the executable where the thunks for our dummy .dll are stored
|
||||
fn find_thunks_start_offset(
|
||||
executable: &[u8],
|
||||
fn find_thunks_start_offset<'a>(
|
||||
executable: &'a [u8],
|
||||
dynamic_relocations: &DynamicRelocationsPe,
|
||||
) -> usize {
|
||||
) -> DummyDllThunks<'a> {
|
||||
// The host executable contains indirect calls to functions that the host should provide
|
||||
//
|
||||
// 14000105d: e8 8e 27 00 00 call 0x1400037f0
|
||||
|
|
@ -854,7 +883,7 @@ fn find_thunks_start_offset(
|
|||
// - https://learn.microsoft.com/en-us/archive/msdn-magazine/2002/february/inside-windows-win32-portable-executable-file-format-in-detail
|
||||
const W: usize = std::mem::size_of::<ImageImportDescriptor>();
|
||||
|
||||
let dummy_import_desc_start = dynamic_relocations.imports_offset_in_file as usize
|
||||
let dummy_import_desc_start = dynamic_relocations.import_directory_offset_in_file as usize
|
||||
+ W * dynamic_relocations.dummy_import_index as usize;
|
||||
|
||||
let dummy_import_desc =
|
||||
|
|
@ -880,19 +909,21 @@ fn find_thunks_start_offset(
|
|||
let sections = nt_headers.sections(executable, offset).unwrap();
|
||||
|
||||
// find the section the virtual address is in
|
||||
let (section_va, offset_in_file) = sections
|
||||
let (section_virtual_address, section) = sections
|
||||
.iter()
|
||||
.find_map(|section| {
|
||||
section
|
||||
.pe_data_containing(executable, dummy_thunks_address)
|
||||
.map(|(_section_data, section_va)| {
|
||||
(section_va, section.pointer_to_raw_data.get(LE))
|
||||
})
|
||||
.map(|(_section_data, section_va)| (section_va, section))
|
||||
})
|
||||
.expect("Invalid thunk virtual address");
|
||||
|
||||
// and get the offset in the file of 0x1400037f0
|
||||
(dummy_thunks_address - section_va + offset_in_file) as usize
|
||||
DummyDllThunks {
|
||||
section,
|
||||
// and get the offset in the file of 0x1400037f0
|
||||
offset_in_file: dummy_thunks_address as u64 - section_virtual_address as u64
|
||||
+ section.pointer_to_raw_data.get(LE) as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Make the thunks point to our actual roc application functions
|
||||
|
|
@ -903,9 +934,10 @@ fn redirect_dummy_dll_functions(
|
|||
thunks_start_offset: usize,
|
||||
) {
|
||||
// it could be that a symbol exposed by the app is not used by the host. We must skip unused symbols
|
||||
let mut targets = function_definition_vas.iter();
|
||||
// this is an O(n^2) loop, hopefully that does not become a problem. If it does we can sort
|
||||
// both vectors to get linear complexity in the loop.
|
||||
'outer: for (i, host_name) in imports.iter().enumerate() {
|
||||
for (roc_app_target_name, roc_app_target_va) in targets.by_ref() {
|
||||
for (roc_app_target_name, roc_app_target_va) in function_definition_vas {
|
||||
if host_name == roc_app_target_name {
|
||||
// addresses are 64-bit values
|
||||
let address_bytes = &mut executable[thunks_start_offset + i * 8..][..8];
|
||||
|
|
@ -1032,15 +1064,13 @@ impl AppSections {
|
|||
|
||||
let address = symbol.as_ref().map(|s| s.address()).unwrap_or_default();
|
||||
let name = symbol.and_then(|s| s.name()).unwrap_or_default();
|
||||
let name = redirect_libc_functions(name).unwrap_or(name).to_string();
|
||||
|
||||
relocations
|
||||
.entry(name.to_string())
|
||||
.or_default()
|
||||
.push(AppRelocation {
|
||||
offset_in_section,
|
||||
address,
|
||||
relocation,
|
||||
});
|
||||
relocations.entry(name).or_default().push(AppRelocation {
|
||||
offset_in_section,
|
||||
address,
|
||||
relocation,
|
||||
});
|
||||
}
|
||||
_ => todo!(),
|
||||
}
|
||||
|
|
@ -1198,12 +1228,17 @@ fn write_section_header(
|
|||
data[section_header_start..][..header_array.len()].copy_from_slice(&header_array);
|
||||
}
|
||||
|
||||
struct BaseRelocations {
|
||||
new_size: u32,
|
||||
delta: u32,
|
||||
}
|
||||
|
||||
fn write_image_base_relocation(
|
||||
mmap: &mut [u8],
|
||||
reloc_section_start: usize,
|
||||
new_block_va: u32,
|
||||
relocations: &[u16],
|
||||
) -> usize {
|
||||
) -> BaseRelocations {
|
||||
// first, collect the blocks of relocations (each relocation block covers a 4K page)
|
||||
// we will be moving stuff around, and have to work from the back to the front. However, the
|
||||
// relocations are encoded in such a way that we can only decode them from front to back.
|
||||
|
|
@ -1228,6 +1263,8 @@ fn write_image_base_relocation(
|
|||
next_block_start += header.size_of_block.get(LE);
|
||||
}
|
||||
|
||||
let old_size = next_block_start - reloc_section_start as u32;
|
||||
|
||||
// this is 8 in practice
|
||||
const HEADER_WIDTH: usize = std::mem::size_of::<ImageBaseRelocation>();
|
||||
|
||||
|
|
@ -1279,7 +1316,10 @@ fn write_image_base_relocation(
|
|||
}
|
||||
}
|
||||
|
||||
shift_amount
|
||||
BaseRelocations {
|
||||
new_size: old_size + shift_amount as u32,
|
||||
delta: shift_amount as u32,
|
||||
}
|
||||
} else {
|
||||
let header =
|
||||
load_struct_inplace_mut::<ImageBaseRelocation>(mmap, next_block_start as usize);
|
||||
|
|
@ -1301,7 +1341,10 @@ fn write_image_base_relocation(
|
|||
// sort by VA. Upper 4 bits store the relocation type
|
||||
entries.sort_unstable_by_key(|x| x & 0b0000_1111_1111_1111);
|
||||
|
||||
size_of_block
|
||||
BaseRelocations {
|
||||
new_size: old_size + size_of_block as u32,
|
||||
delta: size_of_block as u32,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1309,8 +1352,8 @@ fn write_image_base_relocation(
|
|||
/// in a table to find the actual address of the app function. This table must be relocated,
|
||||
/// because it contains absolute addresses to jump to.
|
||||
fn relocate_dummy_dll_entries(executable: &mut [u8], md: &PeMetadata) {
|
||||
let thunks_start_va = (md.rdata_virtual_address - md.image_base as u32)
|
||||
+ md.thunks_start_offset_in_section as u32;
|
||||
let thunks_start_va =
|
||||
md.dummy_dll_thunk_section_virtual_address + md.thunks_start_offset_in_section as u32;
|
||||
|
||||
// relocations are defined per 4kb page
|
||||
const BLOCK_SIZE: u32 = 4096;
|
||||
|
|
@ -1322,7 +1365,7 @@ fn relocate_dummy_dll_entries(executable: &mut [u8], md: &PeMetadata) {
|
|||
.map(|i| (thunks_offset_in_block as usize + 2 * i) as u16)
|
||||
.collect();
|
||||
|
||||
let added_reloc_bytes = write_image_base_relocation(
|
||||
let base_relocations = write_image_base_relocation(
|
||||
executable,
|
||||
md.reloc_offset_in_file,
|
||||
thunks_relocation_block_va,
|
||||
|
|
@ -1343,28 +1386,40 @@ fn relocate_dummy_dll_entries(executable: &mut [u8], md: &PeMetadata) {
|
|||
let reloc_section =
|
||||
load_struct_inplace_mut::<ImageSectionHeader>(executable, reloc_section_header_start);
|
||||
let old_section_size = reloc_section.virtual_size.get(LE);
|
||||
let new_virtual_size = old_section_size + added_reloc_bytes as u32;
|
||||
let new_virtual_size = old_section_size + base_relocations.delta;
|
||||
let new_virtual_size = next_multiple_of(new_virtual_size as usize, 8) as u32;
|
||||
reloc_section.virtual_size.set(LE, new_virtual_size);
|
||||
|
||||
assert!(
|
||||
reloc_section.pointer_to_raw_data.get(LE)
|
||||
+ reloc_section.virtual_size.get(LE)
|
||||
+ (added_reloc_bytes as u32)
|
||||
+ base_relocations.delta
|
||||
< next_section_pointer_to_raw_data,
|
||||
"new .reloc section is too big, and runs into the next section!",
|
||||
);
|
||||
|
||||
// // in the data directories, update the length of the base relocations
|
||||
// let dir = load_struct_inplace_mut::<pe::ImageDataDirectory>(
|
||||
// executable,
|
||||
// md.dynamic_relocations.data_directories_offset_in_file as usize
|
||||
// + object::pe::IMAGE_DIRECTORY_ENTRY_BASERELOC
|
||||
// * std::mem::size_of::<pe::ImageDataDirectory>(),
|
||||
// );
|
||||
//
|
||||
// let old_dir_size = dir.size.get(LE);
|
||||
// debug_assert_eq!(old_section_size, old_dir_size);
|
||||
// dir.size.set(LE, new_virtual_size);
|
||||
// in the data directories, update the length of the base relocations
|
||||
let dir = load_struct_inplace_mut::<pe::ImageDataDirectory>(
|
||||
executable,
|
||||
md.dynamic_relocations.data_directories_offset_in_file as usize
|
||||
+ object::pe::IMAGE_DIRECTORY_ENTRY_BASERELOC
|
||||
* std::mem::size_of::<pe::ImageDataDirectory>(),
|
||||
);
|
||||
|
||||
// it is crucial that the directory size is rounded up to a multiple of 8!
|
||||
// the value is already rounded, so to be correct we can't rely on `dir.size.get(LE)`
|
||||
let new_reloc_directory_size = next_multiple_of(base_relocations.new_size as usize, 8);
|
||||
dir.size.set(LE, new_reloc_directory_size as u32);
|
||||
}
|
||||
|
||||
/// Redirect `memcpy` and similar libc functions to their roc equivalents
|
||||
pub(crate) fn redirect_libc_functions(name: &str) -> Option<&str> {
|
||||
match name {
|
||||
"memcpy" => Some("roc_memcpy"),
|
||||
"memset" => Some("roc_memset"),
|
||||
"memmove" => Some("roc_memmove"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -1554,7 +1609,7 @@ mod test {
|
|||
remove_dummy_dll_import_table_test(
|
||||
&mut data,
|
||||
dynamic_relocations.data_directories_offset_in_file,
|
||||
dynamic_relocations.imports_offset_in_file,
|
||||
dynamic_relocations.import_directory_offset_in_file,
|
||||
dynamic_relocations.dummy_import_index,
|
||||
);
|
||||
|
||||
|
|
@ -1696,14 +1751,14 @@ mod test {
|
|||
// make the dummy dylib based on the app object
|
||||
let names: Vec<_> = symbols.iter().map(|s| s.name.clone()).collect();
|
||||
let dylib_bytes = crate::generate_dylib::synthetic_dll(&names);
|
||||
std::fs::write(dir.join("libapp.obj"), dylib_bytes).unwrap();
|
||||
std::fs::write(dir.join("libapp.dll"), dylib_bytes).unwrap();
|
||||
|
||||
// now we can compile the host (it uses libapp.obj, hence the order here)
|
||||
// now we can compile the host (it uses libapp.dll, hence the order here)
|
||||
let output = std::process::Command::new(&zig)
|
||||
.current_dir(dir)
|
||||
.args(&[
|
||||
"build-exe",
|
||||
"libapp.obj",
|
||||
"libapp.dll",
|
||||
"host.zig",
|
||||
"-lc",
|
||||
"-target",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue