WIP

WIP

WIP

ignore some cli tests that are going to move

try fix for linux valgrind tests

try a legacy host for valgrind

fix roc_glue tests
This commit is contained in:
Luke Boswell 2024-07-02 15:35:07 +10:00
parent b9a17f4a49
commit dc0902bc92
No known key found for this signature in database
GPG key ID: F6DB3C9DB47377B0
12 changed files with 350 additions and 327 deletions

View file

@ -1,5 +1,5 @@
use crate::link::{
legacy_host_file, link, preprocess_host_wasm32, rebuild_host, LinkType, LinkingStrategy,
find_legacy_host, link, preprocess_host_wasm32, rebuild_host, LinkType, LinkingStrategy,
};
use bumpalo::Bump;
use inkwell::memory_buffer::MemoryBuffer;
@ -714,7 +714,7 @@ pub fn build_file<'a>(
emit_timings: bool,
link_type: LinkType,
linking_strategy: LinkingStrategy,
prebuilt_requested: bool,
rebuild_host: bool,
wasm_dev_stack_bytes: Option<u32>,
roc_cache_dir: RocCacheDir<'_>,
load_config: LoadConfig,
@ -740,7 +740,7 @@ pub fn build_file<'a>(
emit_timings,
link_type,
linking_strategy,
prebuilt_requested,
rebuild_host,
wasm_dev_stack_bytes,
loaded,
compilation_start,
@ -756,121 +756,40 @@ fn build_loaded_file<'a>(
code_gen_options: CodeGenOptions,
emit_timings: bool,
link_type: LinkType,
mut linking_strategy: LinkingStrategy,
prebuilt_requested: bool,
linking_strategy: LinkingStrategy,
build_host_requested: bool,
wasm_dev_stack_bytes: Option<u32>,
loaded: roc_load::MonomorphizedModule<'a>,
compilation_start: Instant,
out_path: Option<&Path>,
) -> Result<BuiltFile<'a>, BuildFileError<'a>> {
// get the platform path from the app header
let platform_main_roc = match &loaded.entry_point {
EntryPoint::Executable { platform_path, .. } => platform_path.to_path_buf(),
_ => unreachable!(),
};
// For example, if we're loading the platform from a URL, it's automatically prebuilt
// even if the --prebuilt-platform CLI flag wasn't set.
let is_platform_prebuilt = prebuilt_requested || loaded.uses_prebuilt_platform;
let output_exe_path = get_exe_path(
out_path,
&app_module_path,
target,
linking_strategy,
link_type,
);
if is_platform_prebuilt && linking_strategy == LinkingStrategy::Surgical {
// Fallback to legacy linking if the preprocessed host file does not exist, but a legacy host does exist.
let preprocessed_host_path =
platform_main_roc.with_file_name(roc_linker::preprocessed_host_filename(target));
let legacy_host_path = legacy_host_file(target, &platform_main_roc);
if !preprocessed_host_path.exists() && legacy_host_path.exists() {
linking_strategy = LinkingStrategy::Legacy;
}
}
let dll_stub_symbols =
roc_linker::ExposedSymbols::from_exposed_to_host(&loaded.interns, &loaded.exposed_to_host);
// the preprocessed host is stored beside the platform's main.roc
let preprocessed_host_path = if linking_strategy == LinkingStrategy::Legacy {
if target == Target::Wasm32 {
// when compiling a wasm application, we implicitly assume here that the host is in zig
// and has a file called "host.zig"
platform_main_roc.with_file_name("host.zig")
} else {
legacy_host_file(target, &platform_main_roc)
}
} else {
platform_main_roc.with_file_name(roc_linker::preprocessed_host_filename(target))
};
let output_exe_path = match out_path {
Some(path) => {
// true iff the path ends with a directory separator,
// e.g. '/' on UNIX, '/' or '\\' on Windows
let ends_with_sep = {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
path.as_os_str().as_bytes().ends_with(&[b'/'])
}
#[cfg(windows)]
{
use std::os::windows::ffi::OsStrExt;
let last = path.as_os_str().encode_wide().last();
last == Some(0x002f)// UTF-16 slash
|| last == Some(0x005c) // UTF-16 backslash
}
};
// If you specified a path that ends in in a directory separator, then
// use that directory, but use the app module's filename for the filename.
if ends_with_sep {
let filename = app_module_path.file_name().unwrap_or_default();
with_output_extension(&path.join(filename), target, linking_strategy, link_type)
} else {
path.to_path_buf()
}
}
None => with_output_extension(&app_module_path, target, linking_strategy, link_type),
};
// We don't need to spawn a rebuild thread when using a prebuilt host.
let rebuild_thread = if matches!(link_type, LinkType::Dylib | LinkType::None) {
None
} else if is_platform_prebuilt {
if !preprocessed_host_path.exists() {
invalid_prebuilt_platform(prebuilt_requested, preprocessed_host_path);
std::process::exit(1);
}
if linking_strategy == LinkingStrategy::Surgical {
// Copy preprocessed host to executable location.
// The surgical linker will modify that copy in-place.
std::fs::copy(&preprocessed_host_path, output_exe_path.as_path()).unwrap();
}
None
} else {
// TODO this should probably be moved before load_and_monomorphize.
// To do this we will need to preprocess files just for their exported symbols.
// Also, we should no longer need to do this once we have platforms on
// a package repository, as we can then get prebuilt platforms from there.
let dll_stub_symbols = roc_linker::ExposedSymbols::from_exposed_to_host(
&loaded.interns,
&loaded.exposed_to_host,
);
let join_handle = spawn_rebuild_thread(
code_gen_options.opt_level,
linking_strategy,
platform_main_roc.clone(),
preprocessed_host_path.clone(),
output_exe_path.clone(),
target,
dll_stub_symbols,
);
Some(join_handle)
};
let host_path = get_host_path(
target,
code_gen_options,
emit_timings,
linking_strategy,
build_host_requested,
platform_main_roc.as_path(),
output_exe_path.as_path(),
dll_stub_symbols,
);
let buf = &mut String::with_capacity(1024);
@ -903,31 +822,13 @@ fn build_loaded_file<'a>(
let problems = report_problems_monomorphized(&mut loaded);
let loaded = loaded;
let opt_rebuild_timing = if let Some(rebuild_thread) = rebuild_thread {
if linking_strategy == LinkingStrategy::Additive {
let rebuild_duration = rebuild_thread
.join()
.expect("Failed to (re)build platform.");
if emit_timings && !is_platform_prebuilt {
println!("Finished rebuilding the platform in {rebuild_duration} ms\n");
}
None
} else {
Some(rebuild_thread)
}
} else {
None
};
let (roc_app_bytes, code_gen_timing, expect_metadata) = gen_from_mono_module(
arena,
loaded,
&app_module_path,
target,
code_gen_options,
&preprocessed_host_path,
&host_path,
wasm_dev_stack_bytes,
);
@ -960,24 +861,19 @@ fn build_loaded_file<'a>(
);
}
if let Some(thread) = opt_rebuild_timing {
let rebuild_duration = thread.join().expect("Failed to (re)build platform.");
if emit_timings && !is_platform_prebuilt {
println!("Finished rebuilding the platform in {rebuild_duration} ms\n");
}
}
// Step 2: link the prebuilt platform and compiled app
// link the prebuilt platform and compiled app
let link_start = Instant::now();
match (linking_strategy, link_type) {
(LinkingStrategy::Surgical, _) => {
let metadata_file =
platform_main_roc.with_file_name(roc_linker::metadata_file_name(target));
roc_linker::link_preprocessed_host(
target,
&platform_main_roc,
&roc_app_bytes,
&output_exe_path,
metadata_file,
);
}
(LinkingStrategy::Additive, _) | (LinkingStrategy::Legacy, LinkType::None) => {
@ -1002,36 +898,32 @@ fn build_loaded_file<'a>(
std::fs::write(app_o_file, &*roc_app_bytes).unwrap();
let builtins_host_tempfile = roc_bitcode::host_tempfile()
.expect("failed to write host builtins object to tempfile");
let mut inputs = vec![app_o_file.to_str().unwrap()];
if !matches!(link_type, LinkType::Dylib | LinkType::None) {
// the host has been compiled into a .o or .obj file
inputs.push(preprocessed_host_path.as_path().to_str().unwrap());
inputs.push(host_path.as_path().to_str().unwrap());
}
let builtins_host_tempfile = roc_bitcode::host_tempfile()
.expect("failed to write host builtins object to tempfile");
if matches!(code_gen_options.backend, CodeGenBackend::Assembly(_)) {
inputs.push(builtins_host_tempfile.path().to_str().unwrap());
}
let (mut child, _) = link(target, output_exe_path.clone(), &inputs, link_type)
.map_err(|_| todo!("gracefully handle `ld` failing to spawn."))?;
.map_err(|_| todo!("linker failed to spawn."))?;
let exit_status = child
.wait()
.map_err(|_| todo!("gracefully handle error after `ld` spawned"))?;
.map_err(|_| todo!("linker error after spawning"))?;
// Extend the lifetime of the tempfile so it doesn't get dropped
// (and thus deleted) before the child process is done using it!
let _ = builtins_host_tempfile;
if !exit_status.success() {
todo!(
"gracefully handle `ld` (or `zig` in the case of wasm with --optimize) returning exit code {:?}",
exit_status.code()
);
todo!("linker failed with exit code {:?}", exit_status.code());
}
}
}
@ -1052,124 +944,225 @@ fn build_loaded_file<'a>(
})
}
fn invalid_prebuilt_platform(prebuilt_requested: bool, preprocessed_host_path: PathBuf) {
let prefix = if prebuilt_requested {
"Because I was run with --prebuilt-platform, "
} else {
""
};
/// Get outut path for the executable.
///
/// If you specified a path that ends in in a directory separator, then
/// use that directory, but use the app module's filename for the filename.
fn get_exe_path(
out_path: Option<&Path>,
app_module_path: &PathBuf,
target: Target,
linking_strategy: LinkingStrategy,
link_type: LinkType,
) -> PathBuf {
match out_path {
Some(path) => {
// true iff the path ends with a directory separator,
// e.g. '/' on UNIX, '/' or '\\' on Windows
let ends_with_sep = {
#[cfg(unix)]
{
use std::os::unix::ffi::OsStrExt;
let preprocessed_host_path_str = preprocessed_host_path.to_string_lossy();
let extra_err_msg = if preprocessed_host_path_str.ends_with(".rh") {
"\n\n\tNote: If the platform does have an .rh1 file but no .rh file, it's because it's been built with an older version of roc. Contact the author to release a new build of the platform using a roc release newer than March 21 2023.\n"
} else {
""
};
path.as_os_str().as_bytes().ends_with(&[b'/'])
}
#[cfg(windows)]
{
use std::os::windows::ffi::OsStrExt;
let last = path.as_os_str().encode_wide().last();
last == Some(0x002f)// UTF-16 slash
|| last == Some(0x005c) // UTF-16 backslash
}
};
if ends_with_sep {
let filename = app_module_path.file_name().unwrap_or_default();
with_output_extension(&path.join(filename), target, linking_strategy, link_type)
} else {
path.to_path_buf()
}
}
None => with_output_extension(&app_module_path, target, linking_strategy, link_type),
}
}
fn get_host_path(
target: Target,
code_gen_options: CodeGenOptions,
emit_timings: bool,
linking_strategy: LinkingStrategy,
build_host_requested: bool,
platform_main_roc: &Path,
output_exe_path: &Path,
dll_stub_symbols: Vec<String>,
) -> PathBuf {
let preprocessed_host_path =
platform_main_roc.with_file_name(roc_linker::preprocessed_host_filename(target));
if build_host_requested {
let rebuild_thread = match linking_strategy {
LinkingStrategy::Additive => spawn_wasm32_host_build_thread(
code_gen_options.opt_level,
target,
platform_main_roc.to_owned(),
preprocessed_host_path.to_owned(),
),
LinkingStrategy::Surgical => {
let preprocessed_path = platform_main_roc.with_file_name(format!("{}.rh", target));
let metadata_path =
platform_main_roc.with_file_name(roc_linker::metadata_file_name(target));
spawn_surgical_host_build_thread(
code_gen_options.opt_level,
target,
platform_main_roc.to_owned(),
output_exe_path.to_owned(),
dll_stub_symbols,
preprocessed_path,
metadata_path,
)
}
LinkingStrategy::Legacy => spawn_legacy_host_build_thread(
code_gen_options.opt_level,
target,
platform_main_roc.to_owned(),
),
};
let rebuild_duration = rebuild_thread.join().expect("Failed to build host.");
if emit_timings {
println!(
"Finished rebuilding the platform host in {} ms\n",
rebuild_duration.0
);
}
rebuild_duration.1
} else {
match find_legacy_host(target, platform_main_roc) {
Ok(prebuilt_host) => prebuilt_host,
Err(msg) => {
report_missing_prebuilt_host(msg.as_str());
std::process::exit(1);
}
}
}
}
fn report_missing_prebuilt_host(msg: &str) {
eprintln!(
indoc::indoc!(
r#"
{}I was expecting this file to exist:
I was expecting a prebuilt host to exist:
{}
However, it was not there!{}
However, it was not there!
If you have the platform's source code locally, you may be able to generate it by re-running this command omitting --prebuilt-platform
If you have the platform's source code locally, you may be able to generate it by using a build script.
"#
),
prefix,
preprocessed_host_path.to_string_lossy(),
extra_err_msg
msg
);
}
#[allow(clippy::too_many_arguments)]
fn spawn_rebuild_thread(
fn spawn_wasm32_host_build_thread(
opt_level: OptLevel,
linking_strategy: LinkingStrategy,
platform_main_roc: PathBuf,
preprocessed_host_path: PathBuf,
output_exe_path: PathBuf,
target: Target,
dll_stub_symbols: Vec<String>,
) -> std::thread::JoinHandle<u128> {
platform_main_roc: PathBuf,
output_path: PathBuf,
) -> std::thread::JoinHandle<(u128, PathBuf)> {
std::thread::spawn(move || {
// Printing to stderr because we want stdout to contain only the output of the roc program.
// We are aware of the trade-offs.
// `cargo run` follows the same approach
eprintln!("🔨 Rebuilding platform...");
eprintln!("🔨 Building host ...");
let rebuild_host_start = Instant::now();
let start = Instant::now();
match linking_strategy {
LinkingStrategy::Additive => {
let host_dest = rebuild_host(opt_level, target, platform_main_roc.as_path(), None);
let host_dest = rebuild_host(opt_level, target, platform_main_roc.as_path(), None);
preprocess_host_wasm32(host_dest.as_path(), &preprocessed_host_path);
}
LinkingStrategy::Surgical => {
build_and_preprocess_host_lowlevel(
opt_level,
target,
platform_main_roc.as_path(),
preprocessed_host_path.as_path(),
&dll_stub_symbols,
);
preprocess_host_wasm32(host_dest.as_path(), &output_path);
// Copy preprocessed host to executable location.
// The surgical linker will modify that copy in-place.
std::fs::copy(&preprocessed_host_path, output_exe_path.as_path()).unwrap();
}
LinkingStrategy::Legacy => {
rebuild_host(opt_level, target, platform_main_roc.as_path(), None);
}
}
rebuild_host_start.elapsed().as_millis()
(start.elapsed().as_millis(), output_path)
})
}
pub fn build_and_preprocess_host(
/// Note this will copy the preprocessed host to the executable location
/// where the surgical linker will modify that copy in-place.
fn spawn_surgical_host_build_thread(
opt_level: OptLevel,
target: Target,
platform_main_roc: &Path,
preprocessed_host_path: &Path,
exposed_symbols: roc_linker::ExposedSymbols,
) {
let stub_dll_symbols = exposed_symbols.stub_dll_symbols();
platform_main_roc: PathBuf,
output_exe_path: PathBuf,
dll_stub_symbols: Vec<String>,
preprocessed_path: PathBuf,
metadata_path: PathBuf,
) -> std::thread::JoinHandle<(u128, PathBuf)> {
std::thread::spawn(move || {
// Printing to stderr because we want stdout to contain only the output of the roc program.
// We are aware of the trade-offs.
// `cargo run` follows the same approach
eprintln!("🔨 Building host ...");
build_and_preprocess_host_lowlevel(
opt_level,
target,
platform_main_roc,
preprocessed_host_path,
&stub_dll_symbols,
)
let start = Instant::now();
let stub_lib = roc_linker::generate_stub_lib_from_loaded(
target,
platform_main_roc.as_path(),
dll_stub_symbols.as_slice(),
);
debug_assert!(stub_lib.exists());
let host_exe = rebuild_host(
opt_level,
target,
platform_main_roc.as_path(),
Some(&stub_lib),
);
roc_linker::preprocess_host(
target,
host_exe.as_path(),
metadata_path.as_path(),
preprocessed_path.as_path(),
&stub_lib,
false,
false,
);
// Copy preprocessed host to executable location.
// The surgical linker will modify that copy in-place.
std::fs::copy(&preprocessed_path, output_exe_path.to_owned()).unwrap();
(start.elapsed().as_millis(), output_exe_path)
})
}
fn build_and_preprocess_host_lowlevel(
// Note the output host will be
fn spawn_legacy_host_build_thread(
opt_level: OptLevel,
target: Target,
platform_main_roc: &Path,
_preprocessed_host_path: &Path,
stub_dll_symbols: &[String],
) {
let stub_lib =
roc_linker::generate_stub_lib_from_loaded(target, platform_main_roc, stub_dll_symbols);
platform_main_roc: PathBuf,
) -> std::thread::JoinHandle<(u128, PathBuf)> {
std::thread::spawn(move || {
// Printing to stderr because we want stdout to contain only the output of the roc program.
// We are aware of the trade-offs.
// `cargo run` follows the same approach
eprintln!("🔨 Building host ...");
debug_assert!(stub_lib.exists());
let start = Instant::now();
let host_dest = rebuild_host(opt_level, target, platform_main_roc, Some(&stub_lib));
let host_dest = rebuild_host(opt_level, target, platform_main_roc.as_path(), None);
roc_linker::preprocess_host(
target,
host_dest.as_path(),
platform_main_roc,
&stub_lib,
false,
false,
)
(start.elapsed().as_millis(), host_dest)
})
}
#[allow(clippy::too_many_arguments)]
@ -1255,7 +1248,7 @@ pub fn build_str_test<'a>(
arena: &'a Bump,
app_module_path: &Path,
app_module_source: &'a str,
assume_prebuild: bool,
build_host_requested: bool,
) -> Result<BuiltFile<'a>, BuildFileError<'a>> {
let target = target_lexicon::Triple::host().into();
@ -1300,7 +1293,7 @@ pub fn build_str_test<'a>(
emit_timings,
link_type,
linking_strategy,
assume_prebuild,
build_host_requested,
wasm_dev_stack_bytes,
loaded,
compilation_start,