language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-08-04 12:18:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Merge pull request #4128 from roc-lang/static-relocations

Browse source 
PE inter-section relocations
This commit is contained in:
Folkert de Vries 2022-10-01 00:01:56 +02:00 • committed by GitHub
commit 2b91154b93
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 494 additions and 387 deletions
Download patch file Download diff file Expand all files Collapse all files

328
crates/linker/src/elf.rs
View file

@ -2,9 +2,9 @@ use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind};
use memmap2::MmapMut;
use object::{elf, endian};
use object::{
CompressedFileRange, CompressionFormat, LittleEndian, NativeEndian, Object, ObjectSection,
ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, SectionKind, Symbol,
SymbolIndex, SymbolSection,
CompressedFileRange, CompressionFormat, LittleEndian as LE, NativeEndian, Object,
ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
SectionKind, Symbol, SymbolIndex, SymbolSection,
};
use roc_collections::all::MutMap;
use roc_error_macros::{internal_error, user_error};
@ -15,7 +15,6 @@ use std::mem;
use std::os::raw::c_char;
use std::path::Path;
use std::time::{Duration, Instant};
use target_lexicon::Triple;
use crate::metadata::{self, Metadata, VirtualOffset};
@ -266,7 +265,7 @@ impl<'a> Surgeries<'a> {
/// Constructs a `metadata::Metadata` from a host executable binary, and writes it to disk
pub(crate) fn preprocess_elf(
target: &Triple,
endianness: target_lexicon::Endianness,
host_exe_path: &Path,
metadata_path: &Path,
preprocessed_path: &Path,
@ -410,10 +409,7 @@ pub(crate) fn preprocess_elf(
let scanning_dynamic_deps_duration;
let platform_gen_start;
let out_mmap = match target
.endianness()
.unwrap_or(target_lexicon::Endianness::Little)
{
let out_mmap = match endianness {
target_lexicon::Endianness::Little => {
let scanning_dynamic_deps_start = Instant::now();
@ -510,7 +506,7 @@ fn gen_elf_le(
shared_lib_index: usize,
verbose: bool,
) -> MmapMut {
let exec_header = load_struct_inplace::<elf::FileHeader64<LittleEndian>>(exec_data, 0);
let exec_header = load_struct_inplace::<elf::FileHeader64<LE>>(exec_data, 0);
let ph_offset = exec_header.e_phoff.get(NativeEndian);
let ph_ent_size = exec_header.e_phentsize.get(NativeEndian);
let ph_num = exec_header.e_phnum.get(NativeEndian);
@ -541,7 +537,7 @@ fn gen_elf_le(
out_mmap[..ph_end].copy_from_slice(&exec_data[..ph_end]);
let program_headers = load_structs_inplace_mut::<elf::ProgramHeader64<LittleEndian>>(
let program_headers = load_structs_inplace_mut::<elf::ProgramHeader64<LE>>(
&mut out_mmap,
ph_offset as usize,
ph_num as usize,
@ -572,23 +568,17 @@ fn gen_elf_le(
let p_offset = ph.p_offset.get(NativeEndian);
if (p_type == elf::PT_LOAD && p_offset == 0) || p_type == elf::PT_PHDR {
// Extend length for the first segment and the program header.
ph.p_filesz = endian::U64::new(
LittleEndian,
ph.p_filesz.get(NativeEndian) + md.added_byte_count,
);
ph.p_memsz = endian::U64::new(
LittleEndian,
ph.p_memsz.get(NativeEndian) + md.added_byte_count,
);
ph.p_filesz = endian::U64::new(LE, ph.p_filesz.get(NativeEndian) + md.added_byte_count);
ph.p_memsz = endian::U64::new(LE, ph.p_memsz.get(NativeEndian) + md.added_byte_count);
} else {
// Shift if needed.
if physical_shift_start <= p_offset {
ph.p_offset = endian::U64::new(LittleEndian, p_offset + md.added_byte_count);
ph.p_offset = endian::U64::new(LE, p_offset + md.added_byte_count);
}
let p_vaddr = ph.p_vaddr.get(NativeEndian);
if virtual_shift_start <= p_vaddr {
ph.p_vaddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count);
ph.p_paddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count);
ph.p_vaddr = endian::U64::new(LE, p_vaddr + md.added_byte_count);
ph.p_paddr = endian::U64::new(LE, p_vaddr + md.added_byte_count);
}
}
}
@ -611,7 +601,7 @@ fn gen_elf_le(
.copy_from_slice(&exec_data[physical_shift_start as usize..]);
// Update all sections for shift for extra program headers.
let section_headers = load_structs_inplace_mut::<elf::SectionHeader64<LittleEndian>>(
let section_headers = load_structs_inplace_mut::<elf::SectionHeader64<LE>>(
&mut out_mmap,
sh_offset as usize + md.added_byte_count as usize,
sh_num as usize,
@ -623,10 +613,10 @@ fn gen_elf_le(
let sh_offset = sh.sh_offset.get(NativeEndian);
let sh_addr = sh.sh_addr.get(NativeEndian);
if physical_shift_start <= sh_offset {
sh.sh_offset = endian::U64::new(LittleEndian, sh_offset + md.added_byte_count);
sh.sh_offset = endian::U64::new(LE, sh_offset + md.added_byte_count);
}
if virtual_shift_start <= sh_addr {
sh.sh_addr = endian::U64::new(LittleEndian, sh_addr + md.added_byte_count);
sh.sh_addr = endian::U64::new(LE, sh_addr + md.added_byte_count);
}
// Record every relocation section.
@ -652,34 +642,33 @@ fn gen_elf_le(
// Update all relocations for shift for extra program headers.
for (sec_offset, sec_size) in rel_sections {
let relocations = load_structs_inplace_mut::<elf::Rel64<LittleEndian>>(
let relocations = load_structs_inplace_mut::<elf::Rel64<LE>>(
&mut out_mmap,
sec_offset as usize + md.added_byte_count as usize,
sec_size as usize / mem::size_of::<elf::Rel64<LittleEndian>>(),
sec_size as usize / mem::size_of::<elf::Rel64<LE>>(),
);
for rel in relocations.iter_mut() {
let r_offset = rel.r_offset.get(NativeEndian);
if virtual_shift_start <= r_offset {
rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count);
rel.r_offset = endian::U64::new(LE, r_offset + md.added_byte_count);
}
}
}
for (sec_offset, sec_size) in rela_sections {
let relocations = load_structs_inplace_mut::<elf::Rela64<LittleEndian>>(
let relocations = load_structs_inplace_mut::<elf::Rela64<LE>>(
&mut out_mmap,
sec_offset as usize + md.added_byte_count as usize,
sec_size as usize / mem::size_of::<elf::Rela64<LittleEndian>>(),
sec_size as usize / mem::size_of::<elf::Rela64<LE>>(),
);
for (i, rel) in relocations.iter_mut().enumerate() {
let r_offset = rel.r_offset.get(NativeEndian);
if virtual_shift_start <= r_offset {
rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count);
rel.r_offset = endian::U64::new(LE, r_offset + md.added_byte_count);
// Deal with potential adjusts to absolute jumps.
// TODO: Verify other relocation types.
if rel.r_type(LittleEndian, false) == elf::R_X86_64_RELATIVE {
let r_addend = rel.r_addend.get(LittleEndian);
rel.r_addend
.set(LittleEndian, r_addend + md.added_byte_count as i64);
if rel.r_type(LE, false) == elf::R_X86_64_RELATIVE {
let r_addend = rel.r_addend.get(LE);
rel.r_addend.set(LE, r_addend + md.added_byte_count as i64);
}
}
// If the relocation goes to a roc function, we need to surgically link it and change it to relative.
@ -688,9 +677,9 @@ fn gen_elf_le(
let r_sym = rel.r_sym(NativeEndian, false);
for (name, index) in got_app_syms.iter() {
if *index as u32 == r_sym {
rel.set_r_info(LittleEndian, false, 0, elf::R_X86_64_RELATIVE);
rel.set_r_info(LE, false, 0, elf::R_X86_64_RELATIVE);
let addend_addr = sec_offset as usize
+ i * mem::size_of::<elf::Rela64<LittleEndian>>()
+ i * mem::size_of::<elf::Rela64<LE>>()
// This 16 skips the first 2 fields and gets to the addend field.
+ 16;
md.surgeries
@ -710,7 +699,7 @@ fn gen_elf_le(
// Update dynamic table entries for shift for extra program headers.
let dyn_offset = md.dynamic_section_offset + md.added_byte_count;
let dyns = load_structs_inplace_mut::<elf::Dyn64<LittleEndian>>(
let dyns = load_structs_inplace_mut::<elf::Dyn64<LE>>(
&mut out_mmap,
dyn_offset as usize,
dynamic_lib_count,
@ -749,7 +738,7 @@ fn gen_elf_le(
| elf::DT_VERNEED => {
let d_addr = d.d_val.get(NativeEndian);
if virtual_shift_start <= d_addr {
d.d_val = endian::U64::new(LittleEndian, d_addr + md.added_byte_count);
d.d_val = endian::U64::new(LE, d_addr + md.added_byte_count);
}
}
_ => {}
@ -760,30 +749,30 @@ fn gen_elf_le(
let symtab_offset = md.symbol_table_section_offset + md.added_byte_count;
let symtab_size = md.symbol_table_size as usize;
let symbols = load_structs_inplace_mut::<elf::Sym64<LittleEndian>>(
let symbols = load_structs_inplace_mut::<elf::Sym64<LE>>(
&mut out_mmap,
symtab_offset as usize,
symtab_size / mem::size_of::<elf::Sym64<LittleEndian>>(),
symtab_size / mem::size_of::<elf::Sym64<LE>>(),
);
for sym in symbols {
let addr = sym.st_value.get(NativeEndian);
if virtual_shift_start <= addr {
sym.st_value = endian::U64::new(LittleEndian, addr + md.added_byte_count);
sym.st_value = endian::U64::new(LE, addr + md.added_byte_count);
}
}
// Update all data in the global offset table.
for (offset, size) in got_sections {
let global_offsets = load_structs_inplace_mut::<endian::U64<LittleEndian>>(
let global_offsets = load_structs_inplace_mut::<endian::U64<LE>>(
&mut out_mmap,
*offset + md.added_byte_count as usize,
size / mem::size_of::<endian::U64<LittleEndian>>(),
size / mem::size_of::<endian::U64<LE>>(),
);
for go in global_offsets.iter_mut() {
let go_addr = go.get(NativeEndian);
if physical_shift_start <= go_addr {
go.set(LittleEndian, go_addr + md.added_byte_count);
go.set(LE, go_addr + md.added_byte_count);
}
}
}
@ -801,17 +790,16 @@ fn gen_elf_le(
}
// Update main elf header for extra data.
let mut file_header =
load_struct_inplace_mut::<elf::FileHeader64<LittleEndian>>(&mut out_mmap, 0);
let mut file_header = load_struct_inplace_mut::<elf::FileHeader64<LE>>(&mut out_mmap, 0);
file_header.e_shoff = endian::U64::new(
LittleEndian,
LE,
file_header.e_shoff.get(NativeEndian) + md.added_byte_count,
);
let e_entry = file_header.e_entry.get(NativeEndian);
if virtual_shift_start <= e_entry {
file_header.e_entry = endian::U64::new(LittleEndian, e_entry + md.added_byte_count);
file_header.e_entry = endian::U64::new(LE, e_entry + md.added_byte_count);
}
file_header.e_phnum = endian::U16::new(LittleEndian, ph_num + added_header_count as u16);
file_header.e_phnum = endian::U16::new(LE, ph_num + added_header_count as u16);
out_mmap
}
@ -1068,7 +1056,7 @@ fn surgery_elf_help(
if !elf64 || !litte_endian {
internal_error!("Only 64bit little endian elf currently supported for surgery");
}
let exec_header = load_struct_inplace::<elf::FileHeader64<LittleEndian>>(exec_mmap, 0);
let exec_header = load_struct_inplace::<elf::FileHeader64<LE>>(exec_mmap, 0);
let ph_offset = exec_header.e_phoff.get(NativeEndian);
let ph_ent_size = exec_header.e_phentsize.get(NativeEndian);
@ -1332,7 +1320,7 @@ fn surgery_elf_help(
// Add 2 new sections and segments.
let new_section_count = 2;
offset += new_section_count * sh_ent_size as usize;
let section_headers = load_structs_inplace_mut::<elf::SectionHeader64<LittleEndian>>(
let section_headers = load_structs_inplace_mut::<elf::SectionHeader64<LE>>(
exec_mmap,
new_sh_offset as usize,
sh_num as usize + new_section_count,
@ -1346,39 +1334,39 @@ fn surgery_elf_help(
// set the new rodata section header
section_headers[section_headers.len() - 2] = elf::SectionHeader64 {
sh_name: endian::U32::new(LittleEndian, 0),
sh_type: endian::U32::new(LittleEndian, elf::SHT_PROGBITS),
sh_flags: endian::U64::new(LittleEndian, (elf::SHF_ALLOC) as u64),
sh_addr: endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64),
sh_offset: endian::U64::new(LittleEndian, new_rodata_section_offset as u64),
sh_size: endian::U64::new(LittleEndian, new_rodata_section_size),
sh_link: endian::U32::new(LittleEndian, 0),
sh_info: endian::U32::new(LittleEndian, 0),
sh_addralign: endian::U64::new(LittleEndian, 16),
sh_entsize: endian::U64::new(LittleEndian, 0),
sh_name: endian::U32::new(LE, 0),
sh_type: endian::U32::new(LE, elf::SHT_PROGBITS),
sh_flags: endian::U64::new(LE, elf::SHF_ALLOC as u64),
sh_addr: endian::U64::new(LE, new_rodata_section_vaddr as u64),
sh_offset: endian::U64::new(LE, new_rodata_section_offset as u64),
sh_size: endian::U64::new(LE, new_rodata_section_size),
sh_link: endian::U32::new(LE, 0),
sh_info: endian::U32::new(LE, 0),
sh_addralign: endian::U64::new(LE, 16),
sh_entsize: endian::U64::new(LE, 0),
};
// set the new text section header
section_headers[section_headers.len() - 1] = elf::SectionHeader64 {
sh_name: endian::U32::new(LittleEndian, 0),
sh_type: endian::U32::new(LittleEndian, elf::SHT_PROGBITS),
sh_flags: endian::U64::new(LittleEndian, (elf::SHF_ALLOC | elf::SHF_EXECINSTR) as u64),
sh_addr: endian::U64::new(LittleEndian, new_text_section_vaddr),
sh_offset: endian::U64::new(LittleEndian, new_text_section_offset as u64),
sh_size: endian::U64::new(LittleEndian, new_text_section_size),
sh_link: endian::U32::new(LittleEndian, 0),
sh_info: endian::U32::new(LittleEndian, 0),
sh_addralign: endian::U64::new(LittleEndian, 16),
sh_entsize: endian::U64::new(LittleEndian, 0),
sh_name: endian::U32::new(LE, 0),
sh_type: endian::U32::new(LE, elf::SHT_PROGBITS),
sh_flags: endian::U64::new(LE, (elf::SHF_ALLOC | elf::SHF_EXECINSTR) as u64),
sh_addr: endian::U64::new(LE, new_text_section_vaddr),
sh_offset: endian::U64::new(LE, new_text_section_offset as u64),
sh_size: endian::U64::new(LE, new_text_section_size),
sh_link: endian::U32::new(LE, 0),
sh_info: endian::U32::new(LE, 0),
sh_addralign: endian::U64::new(LE, 16),
sh_entsize: endian::U64::new(LE, 0),
};
// Reload and update file header and size.
let file_header = load_struct_inplace_mut::<elf::FileHeader64<LittleEndian>>(exec_mmap, 0);
file_header.e_shoff = endian::U64::new(LittleEndian, new_sh_offset as u64);
file_header.e_shnum = endian::U16::new(LittleEndian, sh_num + new_section_count as u16);
let file_header = load_struct_inplace_mut::<elf::FileHeader64<LE>>(exec_mmap, 0);
file_header.e_shoff = endian::U64::new(LE, new_sh_offset as u64);
file_header.e_shnum = endian::U16::new(LE, sh_num + new_section_count as u16);
// Add 2 new segments that match the new sections.
let program_headers = load_structs_inplace_mut::<elf::ProgramHeader64<LittleEndian>>(
let program_headers = load_structs_inplace_mut::<elf::ProgramHeader64<LE>>(
exec_mmap,
ph_offset as usize,
ph_num as usize,
@ -1386,27 +1374,27 @@ fn surgery_elf_help(
// set the new rodata section program header
program_headers[program_headers.len() - 2] = elf::ProgramHeader64 {
p_type: endian::U32::new(LittleEndian, elf::PT_LOAD),
p_flags: endian::U32::new(LittleEndian, elf::PF_R),
p_offset: endian::U64::new(LittleEndian, new_rodata_section_offset as u64),
p_vaddr: endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64),
p_paddr: endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64),
p_filesz: endian::U64::new(LittleEndian, new_rodata_section_size),
p_memsz: endian::U64::new(LittleEndian, new_rodata_section_virtual_size),
p_align: endian::U64::new(LittleEndian, md.load_align_constraint),
p_type: endian::U32::new(LE, elf::PT_LOAD),
p_flags: endian::U32::new(LE, elf::PF_R),
p_offset: endian::U64::new(LE, new_rodata_section_offset as u64),
p_vaddr: endian::U64::new(LE, new_rodata_section_vaddr as u64),
p_paddr: endian::U64::new(LE, new_rodata_section_vaddr as u64),
p_filesz: endian::U64::new(LE, new_rodata_section_size),
p_memsz: endian::U64::new(LE, new_rodata_section_virtual_size),
p_align: endian::U64::new(LE, md.load_align_constraint),
};
// set the new text section program header
let new_text_section_index = program_headers.len() - 1;
program_headers[new_text_section_index] = elf::ProgramHeader64 {
p_type: endian::U32::new(LittleEndian, elf::PT_LOAD),
p_flags: endian::U32::new(LittleEndian, elf::PF_R | elf::PF_X),
p_offset: endian::U64::new(LittleEndian, new_text_section_offset as u64),
p_vaddr: endian::U64::new(LittleEndian, new_text_section_vaddr),
p_paddr: endian::U64::new(LittleEndian, new_text_section_vaddr),
p_filesz: endian::U64::new(LittleEndian, new_text_section_size),
p_memsz: endian::U64::new(LittleEndian, new_text_section_size),
p_align: endian::U64::new(LittleEndian, md.load_align_constraint),
p_type: endian::U32::new(LE, elf::PT_LOAD),
p_flags: endian::U32::new(LE, elf::PF_R | elf::PF_X),
p_offset: endian::U64::new(LE, new_text_section_offset as u64),
p_vaddr: endian::U64::new(LE, new_text_section_vaddr),
p_paddr: endian::U64::new(LE, new_text_section_vaddr),
p_filesz: endian::U64::new(LE, new_text_section_size),
p_memsz: endian::U64::new(LE, new_text_section_size),
p_align: endian::U64::new(LE, md.load_align_constraint),
};
// Update calls from platform and dynamic symbols.
@ -1480,14 +1468,14 @@ fn surgery_elf_help(
}
if let Some(i) = md.dynamic_symbol_indices.get(func_name) {
let sym = load_struct_inplace_mut::<elf::Sym64<LittleEndian>>(
let sym = load_struct_inplace_mut::<elf::Sym64<LE>>(
exec_mmap,
dynsym_offset as usize + *i as usize * mem::size_of::<elf::Sym64<LittleEndian>>(),
dynsym_offset as usize + *i as usize * mem::size_of::<elf::Sym64<LE>>(),
);
sym.st_shndx = endian::U16::new(LittleEndian, new_text_section_index as u16);
sym.st_value = endian::U64::new(LittleEndian, func_virt_offset as u64);
sym.st_shndx = endian::U16::new(LE, new_text_section_index as u16);
sym.st_value = endian::U64::new(LE, func_virt_offset as u64);
sym.st_size = endian::U64::new(
LittleEndian,
LE,
match app_func_size_map.get(func_name) {
Some(size) => *size,
None => internal_error!("Size missing for: {func_name}"),
@ -1504,6 +1492,9 @@ fn surgery_elf_help(
mod tests {
use super::*;
use indoc::indoc;
use target_lexicon::Triple;
const ELF64_DYNHOST: &[u8] = include_bytes!("../dynhost_benchmarks_elf64") as &[_];
#[test]
@ -1558,4 +1549,147 @@ mod tests {
keys.as_slice()
)
}
#[allow(dead_code)]
fn zig_host_app_help(dir: &Path) {
let host_zig = indoc!(
r#"
const std = @import("std");
extern fn roc_magic1(usize) callconv(.C) [*]const u8;
pub fn main() !void {
const stdout = std.io.getStdOut().writer();
try stdout.print("Hello {s}\n", .{roc_magic1(0)[0..3]});
}
"#
);
let app_zig = indoc!(
r#"
const X = [_][]const u8 { "foo" };
export fn roc_magic1(index: usize) [*]const u8 {
return X[index].ptr;
}
"#
);
let zig = std::env::var("ROC_ZIG").unwrap_or_else(|_| "zig".into());
std::fs::write(dir.join("host.zig"), host_zig.as_bytes()).unwrap();
std::fs::write(dir.join("app.zig"), app_zig.as_bytes()).unwrap();
// we need to compile the app first
let output = std::process::Command::new(&zig)
.current_dir(dir)
.args(&[
"build-obj",
"app.zig",
"-fPIC",
"-target",
"x86_64-linux-gnu",
"-OReleaseFast",
])
.output()
.unwrap();
if !output.status.success() {
use std::io::Write;
std::io::stdout().write_all(&output.stdout).unwrap();
std::io::stderr().write_all(&output.stderr).unwrap();
panic!("zig build-obj failed");
}
// open our app object; we'll copy sections from it later
let file = std::fs::File::open(dir.join("app.o")).unwrap();
let roc_app = unsafe { memmap2::Mmap::map(&file) }.unwrap();
let names: Vec<String> = {
let object = object::File::parse(&*roc_app).unwrap();
object
.symbols()
.filter(|s| !s.is_local())
.map(|e| e.name().unwrap().to_string())
.collect()
};
let dylib_bytes = crate::generate_dylib::create_dylib_elf64(&names).unwrap();
std::fs::write(dir.join("libapp.so"), dylib_bytes).unwrap();
// now we can compile the host (it uses libapp.obj, hence the order here)
let output = std::process::Command::new(&zig)
.current_dir(dir)
.args(&[
"build-exe",
"libapp.so",
"host.zig",
"-fPIE",
"-lc",
"-target",
"x86_64-linux-gnu",
"-OReleaseFast",
])
.output()
.unwrap();
if !output.status.success() {
use std::io::Write;
std::io::stdout().write_all(&output.stdout).unwrap();
std::io::stderr().write_all(&output.stderr).unwrap();
panic!("zig build-exe failed");
}
preprocess_elf(
target_lexicon::Endianness::Little,
&dir.join("host"),
&dir.join("metadata"),
&dir.join("preprocessedhost"),
&dir.join("libapp.so"),
false,
false,
);
std::fs::copy(&dir.join("preprocessedhost"), &dir.join("final")).unwrap();
surgery_elf(
&*roc_app,
&dir.join("metadata"),
&dir.join("final"),
false,
false,
);
}
#[cfg(target_os = "linux")]
#[test]
fn zig_host_app() {
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
zig_host_app_help(dir);
let output = std::process::Command::new(&dir.join("final"))
.current_dir(dir)
.output()
.unwrap();
if !output.status.success() {
use std::io::Write;
std::io::stdout().write_all(&output.stdout).unwrap();
std::io::stderr().write_all(&output.stderr).unwrap();
panic!("app.exe failed");
}
let output = String::from_utf8_lossy(&output.stdout);
assert_eq!("Hello foo\n", output);
}
}

3
crates/linker/src/generate_dylib/mod.rs
View file

@ -6,6 +6,9 @@ mod pe;
#[cfg(test)]
pub(crate) use pe::synthetic_dll;
#[cfg(test)]
pub(crate) use elf64::create_dylib_elf64;
pub(crate) use pe::APP_DLL;
pub fn generate(target: &Triple, custom_names: &[String]) -> object::read::Result<Vec<u8>> {

33
crates/linker/src/lib.rs
View file

@ -195,10 +195,14 @@ fn preprocess(
println!("Targeting: {}", target);
}
let endianness = target
.endianness()
.unwrap_or(target_lexicon::Endianness::Little);
match target.binary_format {
target_lexicon::BinaryFormat::Elf => {
crate::elf::preprocess_elf(
target,
endianness,
host_exe_path,
metadata_path,
preprocessed_path,
@ -368,3 +372,30 @@ pub(crate) fn open_mmap_mut(path: &Path, length: usize) -> MmapMut {
unsafe { MmapMut::map_mut(&out_file).unwrap_or_else(|e| internal_error!("{e}")) }
}
/// # dbg_hex
/// display dbg result in hexadecimal `{:#x?}` format.
#[macro_export]
macro_rules! dbg_hex {
// NOTE: We cannot use `concat!` to make a static string as a format argument
// of `eprintln!` because `file!` could contain a `{` or
// `$val` expression could be a block (`{ .. }`), in which case the `eprintln!`
// will be malformed.
() => {
eprintln!("[{}:{}]", file!(), line!());
};
($val:expr $(,)?) => {
// Use of `match` here is intentional because it affects the lifetimes
// of temporaries - https://stackoverflow.com/a/48732525/1063961
match $val {
tmp => {
eprintln!("[{}:{}] {} = {:#x?}",
file!(), line!(), stringify!($val), &tmp);
tmp
}
}
};
($($val:expr),+ $(,)?) => {
($($crate::dbg_hex!($val)),+,)
};
}

517
crates/linker/src/pe.rs
View file

@ -16,7 +16,7 @@ use object::{
};
use serde::{Deserialize, Serialize};
use roc_collections::MutMap;
use roc_collections::{MutMap, VecMap};
use roc_error_macros::internal_error;
use crate::{
@ -194,7 +194,7 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
let md = PeMetadata::read_from_file(metadata_path);
let app_obj_sections = AppSections::from_data(roc_app_bytes);
let mut symbols = app_obj_sections.symbols;
let mut symbols = app_obj_sections.roc_symbols;
let image_base: u64 = md.image_base;
let file_alignment = md.file_alignment as usize;
@ -225,6 +225,15 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
let mut data_bytes_added = 0;
let mut file_bytes_added = 0;
// relocations between the sections of the roc application
// (as opposed to relocations for symbols the app imports from the host)
let inter_app_relocations = process_internal_relocations(
&app_obj_sections.sections,
&app_obj_sections.other_symbols,
(app_code_section_va - image_base) as u32,
section_alignment,
);
for kind in [SectionKind::Text, SectionKind::ReadOnlyData] {
let length: usize = app_obj_sections
.sections
@ -287,38 +296,43 @@ pub(crate) fn surgery_pe(executable_path: &Path, metadata_path: &Path, roc_app_b
address,
} = app_relocation;
match md.exports.get(name) {
Some(destination) => {
match relocation.kind() {
object::RelocationKind::Relative => {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
if let Some(destination) = md.exports.get(name) {
match relocation.kind() {
object::RelocationKind::Relative => {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
let delta = destination
- virtual_address as i64
- *offset_in_section as i64
let delta =
destination - virtual_address as i64 - *offset_in_section as i64
+ relocation.addend();
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
}
_ => todo!(),
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
}
_ => todo!(),
}
None => {
match relocation.kind() {
object::RelocationKind::Relative => {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
} else if let Some(destination) = inter_app_relocations.get(name) {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
let delta = *address as i64 - *offset_in_section as i64
+ relocation.addend();
let delta = destination - virtual_address as i64 - *offset_in_section as i64
+ relocation.addend();
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
}
_ => todo!(),
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
} else {
match relocation.kind() {
object::RelocationKind::Relative => {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
let delta =
*address as i64 - *offset_in_section as i64 + relocation.addend();
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
}
_ => todo!(),
}
}
}
@ -767,6 +781,7 @@ fn redirect_dummy_dll_functions(
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[repr(u8)]
enum SectionKind {
Text,
// Data,
@ -786,6 +801,7 @@ struct Section {
file_range: Range<usize>,
kind: SectionKind,
relocations: MutMap<String, AppRelocation>,
app_section_index: SectionIndex,
}
#[derive(Debug)]
@ -798,7 +814,44 @@ struct AppSymbol {
#[derive(Debug, Default)]
struct AppSections {
sections: Vec<Section>,
symbols: Vec<AppSymbol>,
roc_symbols: Vec<AppSymbol>,
other_symbols: Vec<(SectionIndex, AppSymbol)>,
}
/// Process relocations between two places within the app. This a bit different from doing a
/// relocation of a symbol that will be "imported" from the host
fn process_internal_relocations(
sections: &[Section],
other_symbols: &[(SectionIndex, AppSymbol)],
first_host_section_virtual_address: u32,
section_alignment: usize,
) -> VecMap<String, i64> {
let mut result = VecMap::default();
let mut host_section_virtual_address = first_host_section_virtual_address;
for kind in [SectionKind::Text, SectionKind::ReadOnlyData] {
let it = sections.iter().filter(|s| s.kind == kind);
for section in it {
for (s_index, app_symbol) in other_symbols.iter() {
if *s_index == section.app_section_index {
result.insert(
app_symbol.name.clone(),
app_symbol.offset_in_section as i64 + host_section_virtual_address as i64,
);
}
}
}
let length: usize = sections
.iter()
.filter(|s| s.kind == kind)
.map(|s| s.file_range.end - s.file_range.start)
.sum();
host_section_virtual_address += next_multiple_of(length, section_alignment) as u32;
}
result
}
impl AppSections {
@ -866,6 +919,7 @@ impl AppSections {
}
let section = Section {
app_section_index: index,
file_range,
kind,
relocations,
@ -874,7 +928,8 @@ impl AppSections {
sections.push(section);
}
let mut symbols = Vec::new();
let mut roc_symbols = Vec::new();
let mut other_symbols = Vec::new();
for symbol in file.symbols() {
use object::ObjectSymbol;
@ -889,12 +944,26 @@ impl AppSections {
offset_in_section: (offset_in_host_section + symbol.address()) as usize,
};
symbols.push(symbol);
roc_symbols.push(symbol);
}
} else if let object::SymbolSection::Section(index) = symbol.section() {
if let Some((kind, offset_in_host_section)) = section_starts.get(&index) {
let symbol = AppSymbol {
name: symbol.name().unwrap_or_default().to_string(),
section_kind: *kind,
offset_in_section: (offset_in_host_section + symbol.address()) as usize,
};
other_symbols.push((index, symbol));
}
}
}
AppSections { sections, symbols }
AppSections {
sections,
roc_symbols,
other_symbols,
}
}
}
@ -988,9 +1057,6 @@ fn write_section_header(
mod test {
const PE_DYNHOST: &[u8] = include_bytes!("../dynhost_benchmarks_windows.exe") as &[_];
use std::ops::Deref;
use object::pe::ImageFileHeader;
use object::read::pe::PeFile64;
use object::{pe, LittleEndian as LE, Object};
@ -1276,81 +1342,7 @@ mod test {
increase_number_of_sections_help(PE_DYNHOST, &new_sections, &path);
}
fn redirect_dummy_dll_functions_test(
executable: &mut [u8],
dynamic_relocations: &DynamicRelocationsPe,
function_definition_vas: &[(String, u64)],
) {
let object = object::read::pe::PeFile64::parse(&*executable).unwrap();
let imports: Vec<_> = object
.imports()
.unwrap()
.iter()
.filter(|import| import.library() == APP_DLL.as_bytes())
.map(|import| {
std::str::from_utf8(import.name())
.unwrap_or_default()
.to_owned()
})
.collect();
// and get the offset in the file of 0x1400037f0
let thunks_start_offset = find_thunks_start_offset(executable, dynamic_relocations);
redirect_dummy_dll_functions(
executable,
function_definition_vas,
&imports,
thunks_start_offset,
)
}
fn link_zig_host_and_app_help(dir: &Path) {
use object::ObjectSection;
let host_zig = indoc!(
r#"
const std = @import("std");
extern const roc_one: u64;
extern const roc_three: u64;
extern fn roc_magic1() callconv(.C) u64;
extern fn roc_magic2() callconv(.C) u8;
pub export fn roc_alloc() u64 {
return 123456;
}
pub export fn roc_realloc() u64 {
return 111111;
}
pub fn main() !void {
const stdout = std.io.getStdOut().writer();
try stdout.print("Hello, {} {} {} {}!\n", .{roc_magic1(), roc_magic2(), roc_one, roc_three});
}
"#
);
let app_zig = indoc!(
r#"
export const roc_one: u64 = 1;
export const roc_three: u64 = 3;
extern fn roc_alloc() u64;
extern fn roc_realloc() u64;
export fn roc_magic1() u64 {
return roc_alloc() + roc_realloc();
}
export fn roc_magic2() u8 {
return 32;
}
"#
);
fn zig_host_app(dir: &Path, host_zig: &str, app_zig: &str) {
let zig = std::env::var("ROC_ZIG").unwrap_or_else(|_| "zig".into());
std::fs::write(dir.join("host.zig"), host_zig.as_bytes()).unwrap();
@ -1385,7 +1377,7 @@ mod test {
let roc_app = unsafe { memmap2::Mmap::map(&file) }.unwrap();
let roc_app_sections = AppSections::from_data(&*roc_app);
let mut symbols = roc_app_sections.symbols;
let symbols = roc_app_sections.roc_symbols;
// make the dummy dylib based on the app object
let names: Vec<_> = symbols.iter().map(|s| s.name.clone()).collect();
@ -1419,184 +1411,29 @@ mod test {
panic!("zig build-exe failed");
}
let data = std::fs::read(dir.join("host.exe")).unwrap();
let new_sections = [*b".text\0\0\0", *b".rdata\0\0"];
increase_number_of_sections_help(&data, &new_sections, &dir.join("dynhost.exe"));
preprocess_windows(
&dir.join("host.exe"),
&dir.join("metadata"),
&dir.join("preprocessedhost"),
false,
false,
)
.unwrap();
// hardcoded for now, should come from the precompiled metadata in the future
let image_base: u64 = 0x140000000;
let file_alignment = 0x200;
let section_alignment = 0x1000;
let last_host_section_index = 5;
std::fs::copy(&dir.join("preprocessedhost"), &dir.join("app.exe")).unwrap();
let roc_app_sections_size: usize = roc_app_sections
.sections
.iter()
.map(|s| next_multiple_of(s.file_range.end - s.file_range.start, file_alignment))
.sum();
let dynhost_bytes = std::fs::read(dir.join("dynhost.exe")).unwrap();
let mut executable = open_mmap_mut(
&dir.join("app.exe"),
dynhost_bytes.len() + roc_app_sections_size,
);
// copying over all of the dynhost.exe bytes
executable[..dynhost_bytes.len()].copy_from_slice(&dynhost_bytes);
let file = PeFile64::parse(executable.deref()).unwrap();
let last_host_section = file.sections().nth(last_host_section_index).unwrap();
let exports: MutMap<String, i64> = file
.exports()
.unwrap()
.into_iter()
.map(|e| {
(
String::from_utf8(e.name().to_vec()).unwrap(),
(e.address() - image_base) as i64,
)
})
.collect();
let optional_header_offset = file.dos_header().nt_headers_offset() as usize
+ std::mem::size_of::<u32>()
+ std::mem::size_of::<ImageFileHeader>();
let extra_code_section_va = last_host_section.address()
+ next_multiple_of(
last_host_section.size() as usize,
section_alignment as usize,
) as u64;
let mut section_header_start = 624;
let mut section_file_offset = dynhost_bytes.len();
let mut virtual_address = (extra_code_section_va - image_base) as u32;
let mut code_bytes_added = 0;
let mut data_bytes_added = 0;
let mut file_bytes_added = 0;
for kind in [SectionKind::Text, SectionKind::ReadOnlyData] {
let length: usize = roc_app_sections
.sections
.iter()
.filter(|s| s.kind == kind)
.map(|s| s.file_range.end - s.file_range.start)
.sum();
// offset_in_section now becomes a proper virtual address
for symbol in symbols.iter_mut() {
if symbol.section_kind == kind {
symbol.offset_in_section += image_base as usize + virtual_address as usize;
}
}
let virtual_size = length as u32;
let size_of_raw_data = next_multiple_of(length, file_alignment) as u32;
match kind {
SectionKind::Text => {
code_bytes_added += size_of_raw_data;
write_section_header(
&mut executable,
*b".text1\0\0",
pe::IMAGE_SCN_MEM_READ | pe::IMAGE_SCN_CNT_CODE | pe::IMAGE_SCN_MEM_EXECUTE,
section_header_start,
section_file_offset,
virtual_size,
virtual_address,
size_of_raw_data,
);
}
SectionKind::ReadOnlyData => {
data_bytes_added += size_of_raw_data;
write_section_header(
&mut executable,
*b".rdata1\0",
pe::IMAGE_SCN_MEM_READ | pe::IMAGE_SCN_CNT_INITIALIZED_DATA,
section_header_start,
section_file_offset,
virtual_size,
virtual_address,
size_of_raw_data,
);
}
}
let mut offset = section_file_offset;
let it = roc_app_sections.sections.iter().filter(|s| s.kind == kind);
for section in it {
let slice = &roc_app[section.file_range.start..section.file_range.end];
executable[offset..][..slice.len()].copy_from_slice(slice);
for (name, app_relocation) in section.relocations.iter() {
let destination = exports[name];
let AppRelocation {
offset_in_section,
relocation,
..
} = app_relocation;
match relocation.kind() {
object::RelocationKind::Relative => {
// we implicitly only do 32-bit relocations
debug_assert_eq!(relocation.size(), 32);
let delta =
destination - virtual_address as i64 - *offset_in_section as i64
+ relocation.addend();
executable[offset + *offset_in_section as usize..][..4]
.copy_from_slice(&(delta as i32).to_le_bytes());
}
_ => todo!(),
}
}
offset += slice.len();
}
section_header_start += std::mem::size_of::<ImageSectionHeader>();
section_file_offset += size_of_raw_data as usize;
virtual_address += next_multiple_of(length, section_alignment) as u32;
file_bytes_added += next_multiple_of(length, section_alignment) as u32;
}
update_optional_header(
&mut executable,
optional_header_offset,
code_bytes_added as u32,
file_bytes_added as u32,
data_bytes_added as u32,
);
let dynamic_relocations = DynamicRelocationsPe::new(&executable);
let symbols: Vec<_> = symbols
.into_iter()
.map(|s| (s.name, s.offset_in_section as u64))
.collect();
redirect_dummy_dll_functions_test(&mut executable, &dynamic_relocations, &symbols);
remove_dummy_dll_import_table_test(
&mut executable,
dynamic_relocations.data_directories_offset_in_file,
dynamic_relocations.imports_offset_in_file,
dynamic_relocations.dummy_import_index,
);
surgery_pe(&dir.join("app.exe"), &dir.join("metadata"), &*roc_app);
}
#[cfg(windows)]
#[test]
fn link_zig_host_and_app_windows() {
#[allow(dead_code)]
fn windows_test<F>(runner: F) -> String
where
F: Fn(&Path),
{
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
link_zig_host_and_app_help(dir);
runner(dir);
let output = std::process::Command::new("app.exe")
.current_dir(dir)
@ -1612,18 +1449,18 @@ mod test {
panic!("app.exe failed");
}
let output = String::from_utf8_lossy(&output.stdout);
assert_eq!("Hello, 234567 32 1 3!\n", output);
String::from_utf8(output.stdout.to_vec()).unwrap()
}
#[ignore]
#[test]
fn link_zig_host_and_app_wine() {
#[allow(dead_code)]
fn wine_test<F>(runner: F) -> String
where
F: Fn(&Path),
{
let dir = tempfile::tempdir().unwrap();
let dir = dir.path();
link_zig_host_and_app_help(dir);
runner(dir);
let output = std::process::Command::new("wine")
.current_dir(dir)
@ -1640,8 +1477,110 @@ mod test {
panic!("wine failed");
}
let output = String::from_utf8_lossy(&output.stdout);
String::from_utf8(output.stdout.to_vec()).unwrap()
}
assert_eq!("Hello, 234567 32 1 3!\n", output);
/// Basics of linking: symbols imported and exported by the host and app, both values and
/// functions
#[allow(dead_code)]
fn test_basics(dir: &Path) {
zig_host_app(
dir,
indoc!(
r#"
const std = @import("std");
extern const roc_one: u64;
extern const roc_three: u64;
extern fn roc_magic1() callconv(.C) u64;
extern fn roc_magic2() callconv(.C) u8;
pub export fn roc_alloc() u64 {
return 123456;
}
pub export fn roc_realloc() u64 {
return 111111;
}
pub fn main() !void {
const stdout = std.io.getStdOut().writer();
try stdout.print("Hello, {} {} {} {}!\n", .{roc_magic1(), roc_magic2(), roc_one, roc_three});
}
"#
),
indoc!(
r#"
export const roc_one: u64 = 1;
export const roc_three: u64 = 3;
extern fn roc_alloc() u64;
extern fn roc_realloc() u64;
export fn roc_magic1() u64 {
return roc_alloc() + roc_realloc();
}
export fn roc_magic2() u8 {
return 32;
}
"#
),
);
}
#[cfg(windows)]
#[test]
fn basics_windows() {
assert_eq!("Hello, 234567 32 1 3!\n", windows_test(test_basics))
}
#[ignore]
#[test]
fn basics_wine() {
assert_eq!("Hello, 234567 32 1 3!\n", wine_test(test_basics))
}
/// This zig code sample has a relocation in the text section that points into the rodata
/// section. That means we need to correctly track where each app section ends up in the host.
#[allow(dead_code)]
fn test_internal_relocations(dir: &Path) {
zig_host_app(
dir,
indoc!(
r#"
const std = @import("std");
extern fn roc_magic1(usize) callconv(.C) [*]const u8;
pub fn main() !void {
const stdout = std.io.getStdOut().writer();
try stdout.print("Hello {s}\n", .{roc_magic1(0)[0..3]});
}
"#
),
indoc!(
r#"
const X = [_][]const u8 { "foo" };
export fn roc_magic1(index: usize) [*]const u8 {
return X[index].ptr;
}
"#
),
);
}
#[cfg(windows)]
#[test]
fn app_internal_relocations_windows() {
assert_eq!("Hello foo\n", windows_test(test_internal_relocations))
}
#[ignore]
#[test]
fn app_internal_relocations_wine() {
assert_eq!("Hello foo\n", wine_test(test_internal_relocations))
}
}