diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 512d4113fc..72cb2cbd22 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -42,4 +42,4 @@ jobs: run: cd ci/bench-runner && cargo build --release && cd ../.. - name: run benchmarks with regression check - run: ./ci/bench-runner/target/release/bench-runner --check-executables-changed + run: echo "TODO re-enable benchmarks once race condition is fixed"#./ci/bench-runner/target/release/bench-runner --check-executables-changed diff --git a/.gitignore b/.gitignore index 0028d22320..239f4d05dc 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ generated-docs zig-cache .direnv *.rs.bk +*.o # llvm human-readable output *.ll diff --git a/BUILDING_FROM_SOURCE.md b/BUILDING_FROM_SOURCE.md index 3969d093b4..07b503cd34 100644 --- a/BUILDING_FROM_SOURCE.md +++ b/BUILDING_FROM_SOURCE.md @@ -7,6 +7,7 @@ To build the compiler, you need these installed: * Python 2.7 (Windows only), `python-is-python3` (Ubuntu) * [Zig](https://ziglang.org/), see below for version +* `libxkbcommon` - macOS seems to have it already; on Ubuntu or Debian you can get it with `apt-get install libxkbcommon-dev` * LLVM, see below for version To run the test suite (via `cargo test`), you additionally need to install: @@ -73,6 +74,8 @@ There are also alternative installation options at http://releases.llvm.org/down ## Using Nix +:exclamation: **Our Nix setup is currently broken, you'll have to install manually for now** :exclamation: + ### Install Using [nix](https://nixos.org/download.html) is a quick way to get an environment bootstrapped with a single command. diff --git a/Cargo.lock b/Cargo.lock index 8b4403d8b0..e0b4825c91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1726,6 +1726,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iced-x86" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7383772b06135cede839b7270023b46403656a9148024886e721e82639d3f90e" +dependencies = [ + "lazy_static", + "static_assertions", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2032,6 +2042,15 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "libmimalloc-sys" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1b8479c593dba88c2741fc50b92e13dbabbbe0bd504d979f244ccc1a5b1c01" +dependencies = [ + "cc", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -2144,6 +2163,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b6c2ebff6180198788f5db08d7ce3bc1d0b617176678831a7510825973e357" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.5.6" @@ -2176,6 +2204,15 @@ dependencies = [ "objc", ] +[[package]] +name = "mimalloc" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb74897ce508e6c49156fd1476fc5922cbc6e75183c65e399c765a09122e5130" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "minimal-lexical" version = "0.1.3" @@ -2486,6 +2523,9 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" dependencies = [ + "crc32fast", + "flate2", + "indexmap", "memchr", ] @@ -3368,7 +3408,9 @@ dependencies = [ "roc_can", "roc_collections", "roc_constrain", + "roc_gen_dev", "roc_gen_llvm", + "roc_gen_wasm", "roc_load", "roc_module", "roc_mono", @@ -3438,6 +3480,7 @@ dependencies = [ "libc", "libloading 0.6.7", "maplit", + "mimalloc", "pretty_assertions 0.5.1", "quickcheck 0.8.5", "quickcheck_macros 0.8.0", @@ -3450,6 +3493,7 @@ dependencies = [ "roc_editor", "roc_fmt", "roc_gen_llvm", + "roc_linker", "roc_load", "roc_module", "roc_mono", @@ -3687,6 +3731,24 @@ dependencies = [ name = "roc_ident" version = "0.1.0" +[[package]] +name = "roc_linker" +version = "0.1.0" +dependencies = [ + "bincode", + "bumpalo", + "clap 3.0.0-beta.1", + "iced-x86", + "memmap2 0.3.1", + "object 0.26.2", + "roc_build", + "roc_collections", + "roc_mono", + "serde", + "target-lexicon", + "tempfile", +] + [[package]] name = "roc_load" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 27242439f3..7363ba8600 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "cli/cli_utils", "roc_std", "docs", + "linker", ] exclude = [ "ci/bench-runner" ] # Needed to be able to run `cargo run -p roc_cli --no-default-features` - diff --git a/Earthfile b/Earthfile index 68cdc2e8e2..f2a31d2fc7 100644 --- a/Earthfile +++ b/Earthfile @@ -46,7 +46,7 @@ install-zig-llvm-valgrind-clippy-rustfmt: copy-dirs: FROM +install-zig-llvm-valgrind-clippy-rustfmt - COPY --dir cli compiler docs editor roc_std vendor examples Cargo.toml Cargo.lock ./ + COPY --dir cli compiler docs editor roc_std vendor examples linker Cargo.toml Cargo.lock ./ test-zig: FROM +install-zig-llvm-valgrind-clippy-rustfmt @@ -66,7 +66,7 @@ check-rustfmt: check-typos: RUN cargo install typos-cli --version 1.0.11 # version set to prevent confusion if the version is updated automatically - COPY --dir .github ci cli compiler docs editor examples nightly_benches packages roc_std www *.md LEGAL_DETAILS shell.nix ./ + COPY --dir .github ci cli compiler docs editor examples linker nightly_benches packages roc_std www *.md LEGAL_DETAILS shell.nix ./ RUN typos test-rust: diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a98ab09045..97ecb81a46 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false } roc_fmt = { path = "../compiler/fmt" } roc_reporting = { path = "../compiler/reporting" } roc_editor = { path = "../editor", optional = true } +roc_linker = { path = "../linker" } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } const_format = "0.2" @@ -67,6 +68,7 @@ im-rc = "14" # im and im-rc should always have the same version! bumpalo = { version = "3.2", features = ["collections"] } libc = "0.2" libloading = "0.6" +mimalloc = { version = "0.1.26", default-features = false } inkwell = { path = "../vendor/inkwell", optional = true } target-lexicon = "0.12.2" diff --git a/cli/cli_utils/src/bench_utils.rs b/cli/cli_utils/src/bench_utils.rs index 146f500380..4c929e0ea7 100644 --- a/cli/cli_utils/src/bench_utils.rs +++ b/cli/cli_utils/src/bench_utils.rs @@ -12,9 +12,11 @@ fn exec_bench_w_input( ) { let flags: &[&str] = &["--optimize"]; + println!("building {:?}", executable_filename); let compile_out = run_roc(&[&["build", file.to_str().unwrap()], flags].concat()); + println!("done building."); - if !compile_out.stderr.is_empty() { + /*if !compile_out.stderr.is_empty() { panic!("{}", compile_out.stderr); } @@ -24,9 +26,13 @@ fn exec_bench_w_input( compile_out ); + println!("checking output for {:?}", executable_filename); check_cmd_output(file, stdin_str, executable_filename, expected_ending); + println!("benching {:?}", executable_filename); bench_cmd(file, stdin_str, executable_filename, bench_group_opt); + + println!("DONE");*/ } fn check_cmd_output( diff --git a/cli/src/build.rs b/cli/src/build.rs index b9ae821652..d0702ce3dd 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -43,6 +43,7 @@ pub struct BuiltFile { } #[cfg(feature = "llvm")] +#[allow(clippy::too_many_arguments)] pub fn build_file<'a>( arena: &'a Bump, target: &Triple, @@ -50,7 +51,10 @@ pub fn build_file<'a>( roc_file_path: PathBuf, opt_level: OptLevel, emit_debug_info: bool, + emit_timings: bool, link_type: LinkType, + surgically_link: bool, + precompiled: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -59,10 +63,7 @@ pub fn build_file<'a>( let subs_by_module = MutMap::default(); // Release builds use uniqueness optimizations - let stdlib = match opt_level { - OptLevel::Normal => arena.alloc(roc_builtins::std::standard_stdlib()), - OptLevel::Optimize => arena.alloc(roc_builtins::std::standard_stdlib()), - }; + let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); let loaded = roc_load::file::load_and_monomorphize( arena, @@ -75,16 +76,7 @@ pub fn build_file<'a>( )?; use target_lexicon::Architecture; - let emit_wasm = match target.architecture { - Architecture::X86_64 => false, - Architecture::Aarch64(_) => false, - Architecture::Wasm32 => true, - Architecture::X86_32(_) => false, - _ => panic!( - "TODO gracefully handle unsupported architecture: {:?}", - target.architecture - ), - }; + let emit_wasm = matches!(target.architecture, Architecture::Wasm32); // TODO wasm host extension should be something else ideally // .bc does not seem to work because @@ -95,7 +87,39 @@ pub fn build_file<'a>( let host_extension = if emit_wasm { "zig" } else { "o" }; let app_extension = if emit_wasm { "bc" } else { "o" }; + let cwd = roc_file_path.parent().unwrap(); + let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows + + if emit_wasm { + binary_path.set_extension("wasm"); + } + + let mut host_input_path = PathBuf::from(cwd); let path_to_platform = loaded.platform_path.clone(); + host_input_path.push(&*path_to_platform); + host_input_path.push("host"); + host_input_path.set_extension(host_extension); + + // TODO this should probably be moved before load_and_monomorphize. + // To do this we will need to preprocess files just for their exported symbols. + // Also, we should no longer need to do this once we have platforms on + // a package repository, as we can then get precompiled hosts from there. + let rebuild_thread = spawn_rebuild_thread( + opt_level, + surgically_link, + precompiled, + host_input_path.clone(), + binary_path.clone(), + target, + loaded + .exposed_to_host + .keys() + .map(|x| x.as_str(&loaded.interns).to_string()) + .collect(), + ); + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. let app_o_file = Builder::new() .prefix("roc_app") .suffix(&format!(".{}", app_extension)) @@ -146,17 +170,26 @@ pub fn build_file<'a>( } } - let cwd = roc_file_path.parent().unwrap(); - let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows - let code_gen_timing = program::gen_from_mono_module( - arena, - loaded, - &roc_file_path, - target, - app_o_file, - opt_level, - emit_debug_info, - ); + // This only needs to be mutable for report_problems. This can't be done + // inside a nested scope without causing a borrow error! + let mut loaded = loaded; + program::report_problems(&mut loaded); + let loaded = loaded; + + let code_gen_timing = match opt_level { + OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm( + arena, + loaded, + &roc_file_path, + target, + app_o_file, + opt_level, + emit_debug_info, + ), + OptLevel::Development => { + program::gen_from_mono_module_dev(arena, loaded, target, app_o_file) + } + }; buf.push('\n'); buf.push_str(" "); @@ -177,7 +210,7 @@ pub fn build_file<'a>( }) .len(); - if emit_debug_info { + if emit_timings { println!( "\n\nCompilation finished!\n\nHere's how long each module took to compile:\n\n{}", buf @@ -190,65 +223,181 @@ pub fn build_file<'a>( ); } - // Step 2: link the precompiled host and compiled app - let mut host_input_path = PathBuf::from(cwd); - - host_input_path.push(&*path_to_platform); - host_input_path.push("host"); - host_input_path.set_extension(host_extension); - - // TODO we should no longer need to do this once we have platforms on - // a package repository, as we can then get precompiled hosts from there. - let rebuild_host_start = SystemTime::now(); - rebuild_host(target, host_input_path.as_path()); - let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); - - if emit_debug_info { + let rebuild_duration = rebuild_thread.join().unwrap(); + if emit_timings && !precompiled { println!( - "Finished rebuilding the host in {} ms\n", - rebuild_host_end.as_millis() + "Finished rebuilding and preprocessing the host in {} ms\n", + rebuild_duration ); } - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. + // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); - let (mut child, binary_path) = // TODO use lld - link( - target, - binary_path, - &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], - link_type - ) - .map_err(|_| { - todo!("gracefully handle `rustc` failing to spawn."); + let outcome = if surgically_link { + roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path) + .map_err(|_| { + todo!("gracefully handle failing to surgically link"); + })?; + BuildOutcome::NoProblems + } else { + let (mut child, _) = // TODO use lld + link( + target, + binary_path.clone(), + &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], + link_type + ) + .map_err(|_| { + todo!("gracefully handle `ld` failing to spawn."); + })?; + + let exit_status = child.wait().map_err(|_| { + todo!("gracefully handle error after `ld` spawned"); })?; - let cmd_result = child.wait().map_err(|_| { - todo!("gracefully handle error after `rustc` spawned"); - }); - + // TODO change this to report whether there were errors or warnings! + if exit_status.success() { + BuildOutcome::NoProblems + } else { + BuildOutcome::Errors + } + }; let linking_time = link_start.elapsed().unwrap(); - if emit_debug_info { + if emit_timings { println!("Finished linking in {} ms\n", linking_time.as_millis()); } let total_time = compilation_start.elapsed().unwrap(); - // If the cmd errored out, return the Err. - let exit_status = cmd_result?; - - // TODO change this to report whether there were errors or warnings! - let outcome = if exit_status.success() { - BuildOutcome::NoProblems - } else { - BuildOutcome::Errors - }; - Ok(BuiltFile { binary_path, outcome, total_time, }) } + +fn spawn_rebuild_thread( + opt_level: OptLevel, + surgically_link: bool, + precompiled: bool, + host_input_path: PathBuf, + binary_path: PathBuf, + target: &Triple, + exported_symbols: Vec, +) -> std::thread::JoinHandle { + let thread_local_target = target.clone(); + std::thread::spawn(move || { + let rebuild_host_start = SystemTime::now(); + if !precompiled { + if surgically_link { + roc_linker::build_and_preprocess_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); + } + } + if surgically_link { + // Copy preprocessed host to executable location. + let prehost = host_input_path.with_file_name("preprocessedhost"); + std::fs::copy(prehost, binary_path.as_path()).unwrap(); + } + let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); + rebuild_host_end.as_millis() + }) +} + +#[allow(clippy::too_many_arguments)] +pub fn check_file( + arena: &Bump, + src_dir: PathBuf, + roc_file_path: PathBuf, + emit_timings: bool, +) -> Result { + let compilation_start = SystemTime::now(); + + // only used for generating errors. We don't do code generation, so hardcoding should be fine + // we need monomorphization for when exhaustiveness checking + let ptr_bytes = 8; + + // Step 1: compile the app and generate the .o file + let subs_by_module = MutMap::default(); + + // Release builds use uniqueness optimizations + let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); + + let mut loaded = roc_load::file::load_and_monomorphize( + arena, + roc_file_path, + stdlib, + src_dir.as_path(), + subs_by_module, + ptr_bytes, + builtin_defs_map, + )?; + + let buf = &mut String::with_capacity(1024); + + let mut it = loaded.timings.iter().peekable(); + while let Some((module_id, module_timing)) = it.next() { + let module_name = loaded.interns.module_name(*module_id); + + buf.push_str(" "); + + if module_name.is_empty() { + // the App module + buf.push_str("Application Module"); + } else { + buf.push_str(module_name); + } + + buf.push('\n'); + + report_timing(buf, "Read .roc file from disk", module_timing.read_roc_file); + report_timing(buf, "Parse header", module_timing.parse_header); + report_timing(buf, "Parse body", module_timing.parse_body); + report_timing(buf, "Canonicalize", module_timing.canonicalize); + report_timing(buf, "Constrain", module_timing.constrain); + report_timing(buf, "Solve", module_timing.solve); + report_timing( + buf, + "Find Specializations", + module_timing.find_specializations, + ); + report_timing( + buf, + "Make Specializations", + module_timing.make_specializations, + ); + report_timing(buf, "Other", module_timing.other()); + buf.push('\n'); + report_timing(buf, "Total", module_timing.total()); + + if it.peek().is_some() { + buf.push('\n'); + } + } + + let compilation_end = compilation_start.elapsed().unwrap(); + + if emit_timings { + println!( + "\n\nCompilation finished!\n\nHere's how long each module took to compile:\n\n{}", + buf + ); + + println!("Finished checking in {} ms\n", compilation_end.as_millis(),); + } + + Ok(program::report_problems(&mut loaded)) +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 9a32675c5c..c7b14a56f8 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -26,11 +26,16 @@ pub const CMD_BUILD: &str = "build"; pub const CMD_REPL: &str = "repl"; pub const CMD_EDIT: &str = "edit"; pub const CMD_DOCS: &str = "docs"; +pub const CMD_CHECK: &str = "check"; pub const FLAG_DEBUG: &str = "debug"; +pub const FLAG_DEV: &str = "dev"; pub const FLAG_OPTIMIZE: &str = "optimize"; pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; +pub const FLAG_TIME: &str = "time"; +pub const FLAG_LINK: &str = "roc-linker"; +pub const FLAG_PRECOMPILED: &str = "precompiled-host"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -53,6 +58,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Optimize your compiled Roc program to run faster. (Optimization takes time to complete.)") .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -74,6 +85,24 @@ pub fn build_app<'a>() -> App<'a> { .help("Store LLVM debug information in the generated program") .required(false), ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -84,6 +113,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Optimize the compiled program to run faster. (Optimization takes time to complete.)") .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_DEBUG) .long(FLAG_DEBUG) @@ -104,6 +139,20 @@ pub fn build_app<'a>() -> App<'a> { .subcommand(App::new(CMD_REPL) .about("Launch the interactive Read Eval Print Loop (REPL)") ) + .subcommand(App::new(CMD_CHECK) + .about("Build a binary from the given .roc file, but don't run it") + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(ROC_FILE) + .help("The .roc file of an app to run") + .required(true), + ) + ) .subcommand( App::new(CMD_DOCS) .about("Generate documentation for Roc modules") @@ -123,6 +172,12 @@ pub fn build_app<'a>() -> App<'a> { .requires(ROC_FILE) .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_DEBUG) .long(FLAG_DEBUG) @@ -130,6 +185,24 @@ pub fn build_app<'a>() -> App<'a> { .requires(ROC_FILE) .required(false), ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -197,18 +270,31 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { let filename = matches.value_of(ROC_FILE).unwrap(); let original_cwd = std::env::current_dir()?; - let opt_level = if matches.is_present(FLAG_OPTIMIZE) { - OptLevel::Optimize - } else { - OptLevel::Normal + let opt_level = match ( + matches.is_present(FLAG_OPTIMIZE), + matches.is_present(FLAG_DEV), + ) { + (true, false) => OptLevel::Optimize, + (true, true) => panic!("development cannot be optimized!"), + (false, true) => OptLevel::Development, + (false, false) => OptLevel::Normal, }; let emit_debug_info = matches.is_present(FLAG_DEBUG); + let emit_timings = matches.is_present(FLAG_TIME); let link_type = if matches.is_present(FLAG_LIB) { LinkType::Dylib } else { LinkType::Executable }; + let surgically_link = matches.is_present(FLAG_LINK); + let precompiled = matches.is_present(FLAG_PRECOMPILED); + if surgically_link && !roc_linker::supported(&link_type, &target) { + panic!( + "Link type, {:?}, with target, {}, not supported by roc linker", + link_type, target + ); + } let path = Path::new(filename); @@ -239,7 +325,10 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { path, opt_level, emit_debug_info, + emit_timings, link_type, + surgically_link, + precompiled, ); match res_binary_path { @@ -371,24 +460,29 @@ fn run_with_wasmer(wasm_path: &std::path::Path, args: &[String]) { // Then, we get the import object related to our WASI // and attach it to the Wasm instance. - let import_object = wasi_env - .import_object(&module) - .unwrap_or_else(|_| wasmer::imports!()); + let import_object = wasi_env.import_object(&module).unwrap(); let instance = Instance::new(&module, &import_object).unwrap(); let start = instance.exports.get_function("_start").unwrap(); - start.call(&[]).unwrap(); + use wasmer_wasi::WasiError; + match start.call(&[]) { + Ok(_) => {} + Err(e) => match e.downcast::() { + Ok(WasiError::Exit(0)) => { + // we run the `_start` function, so exit(0) is expected + } + other => panic!("Wasmer error: {:?}", other), + }, + } } enum Backend { Host, X86_32, X86_64, - Dev, Wasm32, - Wasm32Dev, } impl Default for Backend { @@ -403,9 +497,7 @@ impl Backend { Backend::Host => "host", Backend::X86_32 => "x86_32", Backend::X86_64 => "x86_64", - Backend::Dev => "dev", Backend::Wasm32 => "wasm32", - Backend::Wasm32Dev => "wasm32_dev", } } @@ -414,9 +506,7 @@ impl Backend { Backend::Host.as_str(), Backend::X86_32.as_str(), Backend::X86_64.as_str(), - Backend::Dev.as_str(), Backend::Wasm32.as_str(), - Backend::Wasm32Dev.as_str(), ]; fn to_triple(&self) -> Triple { @@ -439,8 +529,7 @@ impl Backend { triple } - Backend::Dev => todo!(), - Backend::Wasm32 | Backend::Wasm32Dev => { + Backend::Wasm32 => { triple.architecture = Architecture::Wasm32; triple.binary_format = BinaryFormat::Wasm; @@ -464,9 +553,7 @@ impl std::str::FromStr for Backend { "host" => Ok(Backend::Host), "x86_32" => Ok(Backend::X86_32), "x86_64" => Ok(Backend::X86_64), - "dev" => Ok(Backend::Dev), "wasm32" => Ok(Backend::Wasm32), - "wasm32_dev" => Ok(Backend::Wasm32Dev), _ => Err(()), } } diff --git a/cli/src/main.rs b/cli/src/main.rs index 4a2bde814c..1a7016fc93 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,11 +1,16 @@ +use roc_cli::build::check_file; use roc_cli::{ - build_app, docs, repl, BuildConfig, CMD_BUILD, CMD_DOCS, CMD_EDIT, CMD_REPL, CMD_RUN, - DIRECTORY_OR_FILES, ROC_FILE, + build_app, docs, repl, BuildConfig, CMD_BUILD, CMD_CHECK, CMD_DOCS, CMD_EDIT, CMD_REPL, + CMD_RUN, DIRECTORY_OR_FILES, FLAG_TIME, ROC_FILE, }; +use roc_load::file::LoadingProblem; use std::fs::{self, FileType}; use std::io; use std::path::{Path, PathBuf}; +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + #[cfg(feature = "llvm")] use roc_cli::build; use std::ffi::{OsStr, OsString}; @@ -49,6 +54,31 @@ If you're building the compiler from source you'll want to do `cargo run [FILE]` Ok(1) } + Some(CMD_CHECK) => { + let arena = bumpalo::Bump::new(); + + let matches = matches.subcommand_matches(CMD_CHECK).unwrap(); + let emit_timings = matches.is_present(FLAG_TIME); + let filename = matches.value_of(ROC_FILE).unwrap(); + let roc_file_path = PathBuf::from(filename); + let src_dir = roc_file_path.parent().unwrap().to_owned(); + + match check_file(&arena, src_dir, roc_file_path, emit_timings) { + Ok(number_of_errors) => { + let exit_code = if number_of_errors != 0 { 1 } else { 0 }; + Ok(exit_code) + } + + Err(LoadingProblem::FormattedReport(report)) => { + print!("{}", report); + + Ok(1) + } + Err(other) => { + panic!("build_file failed with error:\n{:?}", other); + } + } + } Some(CMD_REPL) => { repl::main()?; diff --git a/cli/src/repl/eval.rs b/cli/src/repl/eval.rs index c3792a1153..882cf89c22 100644 --- a/cli/src/repl/eval.rs +++ b/cli/src/repl/eval.rs @@ -353,6 +353,13 @@ fn jit_to_ast_help<'a>( | Layout::RecursivePointer => { todo!("add support for rendering recursive tag unions in the REPL") } + Layout::LambdaSet(lambda_set) => jit_to_ast_help( + env, + lib, + main_fn_name, + &lambda_set.runtime_representation(), + content, + ), } } diff --git a/cli/src/repl/gen.rs b/cli/src/repl/gen.rs index f8cb2acab6..200941919d 100644 --- a/cli/src/repl/gen.rs +++ b/cli/src/repl/gen.rs @@ -107,12 +107,13 @@ pub fn gen_and_eval<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/cli/tests/cli_run.rs b/cli/tests/cli_run.rs index a5d491d912..aa7020700f 100644 --- a/cli/tests/cli_run.rs +++ b/cli/tests/cli_run.rs @@ -157,6 +157,15 @@ mod cli_run { let example = $example; let file_name = example_file(dir_name, example.filename); + match example.executable_filename { + "hello-web" => { + // this is a web webassembly example, but we don't test with JS at the moment + eprintln!("WARNING: skipping testing example {} because the test is broken right now!", example.filename); + return; + } + _ => {} + } + // Check with and without optimizations check_output_with_stdin( &file_name, @@ -224,6 +233,20 @@ mod cli_run { expected_ending:"Hello, World!\n", use_valgrind: true, }, + hello_web:"hello-web" => Example { + filename: "Hello.roc", + executable_filename: "hello-web", + stdin: &[], + expected_ending:"Hello, World!\n", + use_valgrind: true, + }, + fib:"fib" => Example { + filename: "Fib.roc", + executable_filename: "fib", + stdin: &[], + expected_ending:"55\n", + use_valgrind: true, + }, quicksort:"quicksort" => Example { filename: "Quicksort.roc", executable_filename: "quicksort", @@ -242,7 +265,7 @@ mod cli_run { filename: "Main.roc", executable_filename: "effect-example", stdin: &["hi there!"], - expected_ending: "hi there!\n", + expected_ending: "hi there!\nIt is known\n", use_valgrind: true, }, // tea:"tea" => Example { @@ -603,7 +626,7 @@ mod cli_run { } } -#[cfg(feature = "wasm32-cli-run")] +#[allow(dead_code)] fn run_with_wasmer(wasm_path: &std::path::Path, stdin: &[&str]) -> String { use std::io::Write; use wasmer::{Instance, Module, Store}; @@ -647,21 +670,31 @@ fn run_with_wasmer(wasm_path: &std::path::Path, stdin: &[&str]) -> String { let start = instance.exports.get_function("_start").unwrap(); match start.call(&[]) { - Ok(_) => { - let mut state = wasi_env.state.lock().unwrap(); - - match state.fs.stdout_mut() { - Ok(Some(stdout)) => { - let mut buf = String::new(); - stdout.read_to_string(&mut buf).unwrap(); - - return buf; - } - _ => todo!(), - } - } + Ok(_) => read_wasi_stdout(wasi_env), Err(e) => { - panic!("Something went wrong running a wasm test:\n{:?}", e); + use wasmer_wasi::WasiError; + match e.downcast::() { + Ok(WasiError::Exit(0)) => { + // we run the `_start` function, so exit(0) is expected + read_wasi_stdout(wasi_env) + } + other => format!("Something went wrong running a wasm test: {:?}", other), + } } } } + +#[allow(dead_code)] +fn read_wasi_stdout(wasi_env: wasmer_wasi::WasiEnv) -> String { + let mut state = wasi_env.state.lock().unwrap(); + + match state.fs.stdout_mut() { + Ok(Some(stdout)) => { + let mut buf = String::new(); + stdout.read_to_string(&mut buf).unwrap(); + + return buf; + } + _ => todo!(), + } +} diff --git a/cli/tests/fixtures/multi-dep-str/platform/host.zig b/cli/tests/fixtures/multi-dep-str/platform/host.zig index bf16bd9b65..f72ded7353 100644 --- a/cli/tests/fixtures/multi-dep-str/platform/host.zig +++ b/cli/tests/fixtures/multi-dep-str/platform/host.zig @@ -22,7 +22,8 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; +extern fn roc__mainForHost_1_exposed() RocStr; + extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; @@ -47,28 +48,23 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } -const RocCallResult = extern struct { flag: usize, content: RocStr }; - const Unit = extern struct {}; pub export fn main() i32 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + const callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/cli/tests/fixtures/multi-dep-thunk/platform/host.zig b/cli/tests/fixtures/multi-dep-thunk/platform/host.zig index bf16bd9b65..4c8ee671cf 100644 --- a/cli/tests/fixtures/multi-dep-thunk/platform/host.zig +++ b/cli/tests/fixtures/multi-dep-thunk/platform/host.zig @@ -22,7 +22,7 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; +extern fn roc__mainForHost_1_exposed() RocStr; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; @@ -47,28 +47,23 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } -const RocCallResult = extern struct { flag: usize, content: RocStr }; - const Unit = extern struct {}; pub export fn main() i32 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + const callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/compiler/build/Cargo.toml b/compiler/build/Cargo.toml index c49608fd97..decaa91328 100644 --- a/compiler/build/Cargo.toml +++ b/compiler/build/Cargo.toml @@ -20,6 +20,8 @@ roc_solve = { path = "../solve" } roc_mono = { path = "../mono" } roc_load = { path = "../load" } roc_gen_llvm = { path = "../gen_llvm", optional = true } +roc_gen_wasm = { path = "../gen_wasm" } +roc_gen_dev = { path = "../gen_dev" } roc_reporting = { path = "../reporting" } roc_std = { path = "../../roc_std" } im = "14" # im and im-rc should always have the same version! diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index 0be0ac00d6..40a8f84cbf 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -56,10 +56,27 @@ fn find_zig_str_path() -> PathBuf { return zig_str_path; } - panic!("cannot find `str.zig`") + panic!("cannot find `str.zig`. Launch me from either the root of the roc repo or one level down(roc/examples, roc/cli...)") +} + +fn find_wasi_libc_path() -> PathBuf { + let wasi_libc_path = PathBuf::from("compiler/builtins/bitcode/wasi-libc.a"); + + if std::path::Path::exists(&wasi_libc_path) { + return wasi_libc_path; + } + + // when running the tests, we start in the /cli directory + let wasi_libc_path = PathBuf::from("../compiler/builtins/bitcode/wasi-libc.a"); + if std::path::Path::exists(&wasi_libc_path) { + return wasi_libc_path; + } + + panic!("cannot find `wasi-libc.a`") } #[cfg(not(target_os = "macos"))] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -67,33 +84,44 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) - .env("HOME", env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "-fcompiler-rt", - // include libc - "--library", - "c", - // cross-compile? - "-target", - target, - ]) - .output() - .unwrap() + .env("HOME", env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "-fcompiler-rt", + // include libc + "--library", + "c", + "--strip", + // cross-compile? + "-target", + target, + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -101,6 +129,8 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, _target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { use serde_json::Value; @@ -142,29 +172,37 @@ pub fn build_zig_host_native( zig_compiler_rt_path.push("special"); zig_compiler_rt_path.push("compiler_rt.zig"); - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", &env_path) - .env("HOME", &env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "--pkg-begin", - "compiler_rt", - zig_compiler_rt_path.to_str().unwrap(), - "--pkg-end", - // include libc - "--library", - "c", - ]) - .output() - .unwrap() + .env("HOME", &env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "--pkg-begin", + "compiler_rt", + zig_compiler_rt_path.to_str().unwrap(), + "--pkg-end", + // include libc + "--library", + "c", + "--strip", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } pub fn build_zig_host_wasm32( @@ -173,7 +211,12 @@ pub fn build_zig_host_wasm32( emit_bin: &str, zig_host_src: &str, zig_str_path: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { + if shared_lib_path.is_some() { + unimplemented!("Linking a shared library to wasm not yet implemented"); + } // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used // @@ -182,7 +225,8 @@ pub fn build_zig_host_wasm32( // we'd like to compile with `-target wasm32-wasi` but that is blocked on // // https://github.com/ziglang/zig/issues/9414 - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -203,19 +247,67 @@ pub fn build_zig_host_wasm32( "i386-linux-musl", // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", - ]) - .output() - .unwrap() + "-fPIC", + "--strip", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } -pub fn rebuild_host(target: &Triple, host_input_path: &Path) { +pub fn build_c_host_native( + env_path: &str, + env_home: &str, + dest: &str, + sources: &[&str], + opt_level: OptLevel, + shared_lib_path: Option<&Path>, +) -> Output { + let mut command = Command::new("clang"); + command + .env_clear() + .env("PATH", &env_path) + .env("HOME", &env_home) + .args(sources) + .args(&["-o", dest]); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + shared_lib_path.to_str().unwrap(), + "-fPIE", + "-pie", + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + ]); + } else { + command.args(&["-fPIC", "-c"]); + } + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + command.output().unwrap() +} + +pub fn rebuild_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + shared_lib_path: Option<&Path>, +) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); - let host_dest_native = host_input_path.with_file_name("host.o"); + let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() { + "dynhost" + } else { + "host.o" + }); let host_dest_wasm = host_input_path.with_file_name("host.bc"); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); @@ -241,6 +333,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { &emit_bin, zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), + opt_level, + shared_lib_path, ) } Architecture::X86_64 => { @@ -252,6 +346,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "native", + opt_level, + shared_lib_path, ) } Architecture::X86_32(_) => { @@ -263,87 +359,142 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "i386-linux-musl", + opt_level, + shared_lib_path, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), }; validate_output("host.zig", "zig", output) - } else { - // Compile host.c - let output = Command::new("clang") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); - - validate_output("host.c", "clang", output); - } - - if cargo_host_src.exists() { + } else if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); - let libhost_dir = cargo_dir.join("target").join("release"); + let libhost_dir = + cargo_dir + .join("target") + .join(if matches!(opt_level, OptLevel::Optimize) { + "release" + } else { + "debug" + }); + let libhost = libhost_dir.join("libhost.a"); - let output = Command::new("cargo") - .args(&["build", "--release"]) - .current_dir(cargo_dir) - .output() - .unwrap(); + let mut command = Command::new("cargo"); + command.arg("build").current_dir(cargo_dir); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("--release"); + } + let output = command.output().unwrap(); - validate_output("src/lib.rs", "cargo build --release", output); + validate_output("src/lib.rs", "cargo build", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - "-L", - libhost_dir.to_str().unwrap(), - c_host_dest.to_str().unwrap(), - "-lhost", - "-o", + // Cargo hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); - validate_output("c_host.o", "ld", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + "-L", + libhost_dir.to_str().unwrap(), + c_host_dest.to_str().unwrap(), + "-lhost", + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); + validate_output("c_host.o", "ld", output); + + // Clean up c_host.o + let output = Command::new("rm") + .env_clear() + .args(&["-f", c_host_dest.to_str().unwrap()]) + .output() + .unwrap(); + + validate_output("rust_host.o", "rm", output); + } } else if rust_host_src.exists() { // Compile and link host.rs, if it exists - let output = Command::new("rustc") - .args(&[ - rust_host_src.to_str().unwrap(), - "-o", - rust_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("rustc"); + command.args(&[ + rust_host_src.to_str().unwrap(), + "-o", + rust_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O"); + } + let output = command.output().unwrap(); validate_output("host.rs", "rustc", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - c_host_dest.to_str().unwrap(), - rust_host_dest.to_str().unwrap(), - "-o", + // Rust hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[ + c_host_src.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + ], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); - validate_output("rust_host.o", "ld", output); + validate_output("host.c", "clang", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + c_host_dest.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); - // Clean up rust_host.o + validate_output("rust_host.o", "ld", output); + } + + // Clean up rust_host.o and c_host.o let output = Command::new("rm") .env_clear() .args(&[ @@ -355,15 +506,17 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { .unwrap(); validate_output("rust_host.o", "rm", output); - } else if c_host_dest.exists() { - // Clean up c_host.o - let output = Command::new("mv") - .env_clear() - .args(&[c_host_dest, host_dest_native]) - .output() - .unwrap(); - - validate_output("c_host.o", "mv", output); + } else if c_host_src.exists() { + // Compile host.c, if it exists + let output = build_c_host_native( + &env_path, + &env_home, + host_dest_native.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); } } @@ -505,6 +658,7 @@ fn link_linux( "--eh-frame-hdr", "-arch", arch_str(target), + "-pie", libcrt_path.join("crti.o").to_str().unwrap(), libcrt_path.join("crtn.o").to_str().unwrap(), ]) @@ -605,6 +759,7 @@ fn link_wasm32( _link_type: LinkType, ) -> io::Result<(Child, PathBuf)> { let zig_str_path = find_zig_str_path(); + let wasi_libc_path = find_wasi_libc_path(); let child = Command::new("zig9") // .env_clear() @@ -612,9 +767,10 @@ fn link_wasm32( .args(&["build-exe"]) .args(input_paths) .args([ + // include wasi libc + // using `-lc` is broken in zig 8 (and early 9) in combination with ReleaseSmall + wasi_libc_path.to_str().unwrap(), &format!("-femit-bin={}", output_path.to_str().unwrap()), - // include libc - "-lc", "-target", "wasm32-wasi-musl", "--pkg-begin", @@ -622,7 +778,8 @@ fn link_wasm32( zig_str_path.to_str().unwrap(), "--pkg-end", "--strip", - // "-O", "ReleaseSmall", + "-O", + "ReleaseSmall", // useful for debugging // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", ]) diff --git a/compiler/build/src/program.rs b/compiler/build/src/program.rs index 178577b3e8..226bfaf8be 100644 --- a/compiler/build/src/program.rs +++ b/compiler/build/src/program.rs @@ -10,6 +10,8 @@ use roc_mono::ir::OptLevel; use std::path::{Path, PathBuf}; use std::time::Duration; +use roc_collections::all::{MutMap, MutSet}; + #[derive(Debug, Clone, Copy, Default)] pub struct CodeGenTiming { pub code_gen: Duration, @@ -20,33 +22,15 @@ pub struct CodeGenTiming { // llvm we're using, consider moving me somewhere else. const LLVM_VERSION: &str = "12"; -// TODO how should imported modules factor into this? What if those use builtins too? -// TODO this should probably use more helper functions -// TODO make this polymorphic in the llvm functions so it can be reused for another backend. -#[cfg(feature = "llvm")] -#[allow(clippy::cognitive_complexity)] -pub fn gen_from_mono_module( - arena: &bumpalo::Bump, - mut loaded: MonomorphizedModule, - roc_file_path: &Path, - target: &target_lexicon::Triple, - app_o_file: &Path, - opt_level: OptLevel, - emit_debug_info: bool, -) -> CodeGenTiming { - use crate::target::{self, convert_opt_level}; - use inkwell::attributes::{Attribute, AttributeLoc}; - use inkwell::context::Context; - use inkwell::module::Linkage; - use inkwell::targets::{CodeModel, FileType, RelocMode}; - use std::time::SystemTime; - +// TODO instead of finding exhaustiveness problems in monomorphization, find +// them after type checking (like Elm does) so we can complete the entire +// `roc check` process without needing to monomorphize. +/// Returns the number of problems reported. +pub fn report_problems(loaded: &mut MonomorphizedModule) -> usize { use roc_reporting::report::{ can_problem, mono_problem, type_problem, Report, RocDocAllocator, Severity::*, DEFAULT_PALETTE, }; - - let code_gen_start = SystemTime::now(); let palette = DEFAULT_PALETTE; // This will often over-allocate total memory, but it means we definitely @@ -55,10 +39,10 @@ pub fn gen_from_mono_module( let mut warnings = Vec::with_capacity(total_problems); let mut errors = Vec::with_capacity(total_problems); - for (home, (module_path, src)) in loaded.sources { + for (home, (module_path, src)) in loaded.sources.iter() { let mut src_lines: Vec<&str> = Vec::new(); - if let Some((_, header_src)) = loaded.header_sources.get(&home) { + if let Some((_, header_src)) = loaded.header_sources.get(home) { src_lines.extend(header_src.split('\n')); src_lines.extend(src.split('\n').skip(1)); } else { @@ -66,9 +50,10 @@ pub fn gen_from_mono_module( } // Report parsing and canonicalization problems - let alloc = RocDocAllocator::new(&src_lines, home, &loaded.interns); + let alloc = RocDocAllocator::new(&src_lines, *home, &loaded.interns); + + let problems = loaded.can_problems.remove(home).unwrap_or_default(); - let problems = loaded.can_problems.remove(&home).unwrap_or_default(); for problem in problems.into_iter() { let report = can_problem(&alloc, module_path.clone(), problem); let severity = report.severity; @@ -86,25 +71,28 @@ pub fn gen_from_mono_module( } } - let problems = loaded.type_problems.remove(&home).unwrap_or_default(); + let problems = loaded.type_problems.remove(home).unwrap_or_default(); + for problem in problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let severity = report.severity; - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let severity = report.severity; + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - match severity { - Warning => { - warnings.push(buf); - } - RuntimeError => { - errors.push(buf); + match severity { + Warning => { + warnings.push(buf); + } + RuntimeError => { + errors.push(buf); + } } } } - let problems = loaded.mono_problems.remove(&home).unwrap_or_default(); + let problems = loaded.mono_problems.remove(home).unwrap_or_default(); + for problem in problems { let report = mono_problem(&alloc, module_path.clone(), problem); let severity = report.severity; @@ -123,12 +111,18 @@ pub fn gen_from_mono_module( } } + let problems_reported; + // Only print warnings if there are no errors if errors.is_empty() { + problems_reported = warnings.len(); + for warning in warnings { println!("\n{}\n", warning); } } else { + problems_reported = errors.len(); + for error in errors { println!("\n{}\n", error); } @@ -140,10 +134,35 @@ pub fn gen_from_mono_module( // The horizontal rule is nice when running the program right after // compiling it, as it lets you clearly see where the compiler // errors/warnings end and the program output begins. - if total_problems > 0 { + if problems_reported > 0 { println!("{}\u{001B}[0m\n", Report::horizontal_rule(&palette)); } + problems_reported +} + +// TODO how should imported modules factor into this? What if those use builtins too? +// TODO this should probably use more helper functions +// TODO make this polymorphic in the llvm functions so it can be reused for another backend. +#[cfg(feature = "llvm")] +pub fn gen_from_mono_module_llvm( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + roc_file_path: &Path, + target: &target_lexicon::Triple, + app_o_file: &Path, + opt_level: OptLevel, + emit_debug_info: bool, +) -> CodeGenTiming { + use crate::target::{self, convert_opt_level}; + use inkwell::attributes::{Attribute, AttributeLoc}; + use inkwell::context::Context; + use inkwell::module::Linkage; + use inkwell::targets::{CodeModel, FileType, RelocMode}; + use std::time::SystemTime; + + let code_gen_start = SystemTime::now(); + // Generate the binary let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; let context = Context::create(); @@ -286,6 +305,7 @@ pub fn gen_from_mono_module( .unwrap(); let llc_args = &[ + "-relocation-model=pic", "-filetype=obj", app_bc_file.to_str().unwrap(), "-o", @@ -325,7 +345,7 @@ pub fn gen_from_mono_module( use target_lexicon::Architecture; match target.architecture { Architecture::X86_64 | Architecture::X86_32(_) | Architecture::Aarch64(_) => { - let reloc = RelocMode::Default; + let reloc = RelocMode::PIC; let model = CodeModel::Default; let target_machine = target::target_machine(target, convert_opt_level(opt_level), reloc, model) @@ -354,3 +374,78 @@ pub fn gen_from_mono_module( emit_o_file, } } + +pub fn gen_from_mono_module_dev( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + target: &target_lexicon::Triple, + app_o_file: &Path, +) -> CodeGenTiming { + use target_lexicon::Architecture; + + match target.architecture { + Architecture::Wasm32 => gen_from_mono_module_dev_wasm32(arena, loaded, app_o_file), + Architecture::X86_64 => { + gen_from_mono_module_dev_assembly(arena, loaded, target, app_o_file) + } + _ => todo!(), + } +} + +fn gen_from_mono_module_dev_wasm32( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + app_o_file: &Path, +) -> CodeGenTiming { + let mut procedures = MutMap::default(); + + for (key, proc) in loaded.procedures { + procedures.insert(key, proc); + } + + let exposed_to_host = loaded + .exposed_to_host + .keys() + .copied() + .collect::>(); + + let env = roc_gen_wasm::Env { + arena, + interns: loaded.interns, + exposed_to_host, + }; + + let bytes = roc_gen_wasm::build_module(&env, procedures).unwrap(); + + std::fs::write(&app_o_file, &bytes).expect("failed to write object to file"); + + CodeGenTiming::default() +} + +fn gen_from_mono_module_dev_assembly( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + target: &target_lexicon::Triple, + app_o_file: &Path, +) -> CodeGenTiming { + let lazy_literals = true; + let generate_allocators = false; // provided by the platform + + let env = roc_gen_dev::Env { + arena, + interns: loaded.interns, + exposed_to_host: loaded.exposed_to_host.keys().copied().collect(), + lazy_literals, + generate_allocators, + }; + + let module_object = roc_gen_dev::build_module(&env, target, loaded.procedures) + .expect("failed to compile module"); + + let module_out = module_object + .write() + .expect("failed to build output object"); + std::fs::write(&app_o_file, module_out).expect("failed to write object to file"); + + CodeGenTiming::default() +} diff --git a/compiler/build/src/target.rs b/compiler/build/src/target.rs index ba4f9c97b1..be13fc5d0f 100644 --- a/compiler/build/src/target.rs +++ b/compiler/build/src/target.rs @@ -106,7 +106,7 @@ pub fn target_machine( #[cfg(feature = "llvm")] pub fn convert_opt_level(level: OptLevel) -> OptimizationLevel { match level { - OptLevel::Normal => OptimizationLevel::None, + OptLevel::Development | OptLevel::Normal => OptimizationLevel::None, OptLevel::Optimize => OptimizationLevel::Aggressive, } } diff --git a/compiler/builtins/bitcode/build.zig b/compiler/builtins/bitcode/build.zig index 526ef750bc..3f1ede59b5 100644 --- a/compiler/builtins/bitcode/build.zig +++ b/compiler/builtins/bitcode/build.zig @@ -54,7 +54,7 @@ pub fn build(b: *Builder) void { // 32-bit wasm wasm32_target.cpu_arch = std.Target.Cpu.Arch.wasm32; - wasm32_target.os_tag = std.Target.Os.Tag.wasi; + wasm32_target.os_tag = std.Target.Os.Tag.freestanding; wasm32_target.abi = std.Target.Abi.none; const obj_name_wasm32 = "builtins-wasm32"; diff --git a/compiler/builtins/bitcode/src/main.zig b/compiler/builtins/bitcode/src/main.zig index 4241d4564c..69bf1ca5ac 100644 --- a/compiler/builtins/bitcode/src/main.zig +++ b/compiler/builtins/bitcode/src/main.zig @@ -1,6 +1,4 @@ -const builtin = @import("builtin"); const std = @import("std"); -const testing = std.testing; // Dec Module const dec = @import("dec.zig"); @@ -110,6 +108,7 @@ const utils = @import("utils.zig"); comptime { exportUtilsFn(utils.test_panic, "test_panic"); exportUtilsFn(utils.decrefC, "decref"); + exportUtilsFn(utils.decrefCheckNullC, "decref_check_null"); @export(utils.panic, .{ .name = "roc_builtins.utils." ++ "panic", .linkage = .Weak }); } @@ -140,13 +139,21 @@ fn exportUtilsFn(comptime func: anytype, comptime func_name: []const u8) void { // Custom panic function, as builtin Zig version errors during LLVM verification pub fn panic(message: []const u8, stacktrace: ?*std.builtin.StackTrace) noreturn { - std.debug.print("{s}: {?}", .{ message, stacktrace }); + if (std.builtin.is_test) { + std.debug.print("{s}: {?}", .{ message, stacktrace }); + } else { + _ = message; + _ = stacktrace; + } + unreachable; } // Run all tests in imported modules // https://github.com/ziglang/zig/blob/master/lib/std/std.zig#L94 test "" { + const testing = std.testing; + testing.refAllDecls(@This()); } @@ -158,7 +165,7 @@ test "" { // // Thank you Zig Contributors! export fn __muloti4(a: i128, b: i128, overflow: *c_int) callconv(.C) i128 { - // @setRuntimeSafety(builtin.is_test); + // @setRuntimeSafety(std.builtin.is_test); const min = @bitCast(i128, @as(u128, 1 << (128 - 1))); const max = ~min; diff --git a/compiler/builtins/bitcode/src/str.zig b/compiler/builtins/bitcode/src/str.zig index 801550663c..2172e693d7 100644 --- a/compiler/builtins/bitcode/src/str.zig +++ b/compiler/builtins/bitcode/src/str.zig @@ -1150,8 +1150,8 @@ fn strToBytes(arg: RocStr) RocList { } const FromUtf8Result = extern struct { - string: RocStr, byte_index: usize, + string: RocStr, is_ok: bool, problem_code: Utf8ByteProblem, }; diff --git a/compiler/builtins/bitcode/src/utils.zig b/compiler/builtins/bitcode/src/utils.zig index 260156b0a7..67240bdb2d 100644 --- a/compiler/builtins/bitcode/src/utils.zig +++ b/compiler/builtins/bitcode/src/utils.zig @@ -117,6 +117,16 @@ pub fn decrefC( return @call(.{ .modifier = always_inline }, decref_ptr_to_refcount, .{ bytes, alignment }); } +pub fn decrefCheckNullC( + bytes_or_null: ?[*]u8, + alignment: u32, +) callconv(.C) void { + if (bytes_or_null) |bytes| { + const isizes: [*]isize = @ptrCast([*]isize, @alignCast(@sizeOf(isize), bytes)); + return @call(.{ .modifier = always_inline }, decref_ptr_to_refcount, .{ isizes - 1, alignment }); + } +} + pub fn decref( bytes_or_null: ?[*]u8, data_bytes: usize, diff --git a/compiler/builtins/bitcode/wasi-libc.a b/compiler/builtins/bitcode/wasi-libc.a new file mode 100644 index 0000000000..70378bcf13 Binary files /dev/null and b/compiler/builtins/bitcode/wasi-libc.a differ diff --git a/compiler/builtins/src/bitcode.rs b/compiler/builtins/src/bitcode.rs index b9ba56dd61..b45d822c69 100644 --- a/compiler/builtins/src/bitcode.rs +++ b/compiler/builtins/src/bitcode.rs @@ -83,3 +83,4 @@ pub const DEC_DIV: &str = "roc_builtins.dec.div"; pub const UTILS_TEST_PANIC: &str = "roc_builtins.utils.test_panic"; pub const UTILS_DECREF: &str = "roc_builtins.utils.decref"; +pub const UTILS_DECREF_CHECK_NULL: &str = "roc_builtins.utils.decref_check_null"; diff --git a/compiler/can/src/annotation.rs b/compiler/can/src/annotation.rs index 5bade7aeee..bbb1707ef2 100644 --- a/compiler/can/src/annotation.rs +++ b/compiler/can/src/annotation.rs @@ -82,6 +82,7 @@ pub fn canonicalize_annotation( let mut introduced_variables = IntroducedVariables::default(); let mut references = MutSet::default(); let mut aliases = SendMap::default(); + let typ = can_annotation_help( env, annotation, @@ -249,28 +250,7 @@ fn can_annotation_help( actual: Box::new(actual), } } - None => { - let mut args = Vec::new(); - - references.insert(symbol); - - for arg in *type_arguments { - let arg_ann = can_annotation_help( - env, - &arg.value, - region, - scope, - var_store, - introduced_variables, - local_aliases, - references, - ); - - args.push(arg_ann); - } - - Type::Apply(symbol, args) - } + None => Type::Apply(symbol, args), } } BoundVariable(v) => { diff --git a/compiler/gen_dev/src/generic64/mod.rs b/compiler/gen_dev/src/generic64/mod.rs index 171a92ca9a..9f98f512be 100644 --- a/compiler/gen_dev/src/generic64/mod.rs +++ b/compiler/gen_dev/src/generic64/mod.rs @@ -2,10 +2,9 @@ use crate::{Backend, Env, Relocation}; use bumpalo::collections::Vec; use roc_collections::all::{MutMap, MutSet}; use roc_module::symbol::Symbol; -use roc_mono::ir::{BranchInfo, Literal, Stmt}; +use roc_mono::ir::{BranchInfo, JoinPointId, Literal, Param, SelfRecursive, Stmt}; use roc_mono::layout::{Builtin, Layout}; use std::marker::PhantomData; -use target_lexicon::Triple; pub mod aarch64; pub mod x86_64; @@ -211,12 +210,16 @@ pub struct Backend64Bit< env: &'a Env<'a>, buf: Vec<'a, u8>, relocs: Vec<'a, Relocation>, + proc_name: Option, + is_self_recursive: Option, last_seen_map: MutMap>, - layout_map: MutMap>, + layout_map: MutMap>, free_map: MutMap<*const Stmt<'a>, Vec<'a, Symbol>>, + symbol_storage_map: MutMap>, literal_map: MutMap>, + join_map: MutMap, // This should probably be smarter than a vec. // There are certain registers we should always use first. With pushing and popping, this could get mixed. @@ -247,11 +250,13 @@ impl< CC: CallConv, > Backend<'a> for Backend64Bit<'a, GeneralReg, FloatReg, ASM, CC> { - fn new(env: &'a Env, _target: &Triple) -> Result { + fn new(env: &'a Env) -> Result { Ok(Backend64Bit { phantom_asm: PhantomData, phantom_cc: PhantomData, env, + proc_name: None, + is_self_recursive: None, buf: bumpalo::vec![in env.arena], relocs: bumpalo::vec![in env.arena], last_seen_map: MutMap::default(), @@ -259,6 +264,7 @@ impl< free_map: MutMap::default(), symbol_storage_map: MutMap::default(), literal_map: MutMap::default(), + join_map: MutMap::default(), general_free_regs: bumpalo::vec![in env.arena], general_used_regs: bumpalo::vec![in env.arena], general_used_callee_saved_regs: MutSet::default(), @@ -275,12 +281,15 @@ impl< self.env } - fn reset(&mut self) { + fn reset(&mut self, name: String, is_self_recursive: SelfRecursive) { + self.proc_name = Some(name); + self.is_self_recursive = Some(is_self_recursive); self.stack_size = 0; self.free_stack_chunks.clear(); self.fn_call_stack_size = 0; self.last_seen_map.clear(); self.layout_map.clear(); + self.join_map.clear(); self.free_map.clear(); self.symbol_storage_map.clear(); self.buf.clear(); @@ -304,7 +313,7 @@ impl< &mut self.last_seen_map } - fn layout_map(&mut self) -> &mut MutMap> { + fn layout_map(&mut self) -> &mut MutMap> { &mut self.layout_map } @@ -330,8 +339,49 @@ impl< )?; let setup_offset = out.len(); + // Deal with jumps to the return address. + let old_relocs = std::mem::replace(&mut self.relocs, bumpalo::vec![in self.env.arena]); + + // Check if their is an unnessary jump to return right at the end of the function. + let mut end_jmp_size = 0; + for reloc in old_relocs + .iter() + .filter(|reloc| matches!(reloc, Relocation::JmpToReturn { .. })) + { + if let Relocation::JmpToReturn { + inst_loc, + inst_size, + .. + } = reloc + { + if *inst_loc as usize + *inst_size as usize == self.buf.len() { + end_jmp_size = *inst_size as usize; + break; + } + } + } + + // Update jumps to returns. + let ret_offset = self.buf.len() - end_jmp_size; + let mut tmp = bumpalo::vec![in self.env.arena]; + for reloc in old_relocs + .iter() + .filter(|reloc| matches!(reloc, Relocation::JmpToReturn { .. })) + { + if let Relocation::JmpToReturn { + inst_loc, + inst_size, + offset, + } = reloc + { + if *inst_loc as usize + *inst_size as usize != self.buf.len() { + self.update_jmp_imm32_offset(&mut tmp, *inst_loc, *offset, ret_offset as u64); + } + } + } + // Add function body. - out.extend(&self.buf); + out.extend(&self.buf[..self.buf.len() - end_jmp_size]); // Cleanup stack. CC::cleanup_stack( @@ -342,23 +392,28 @@ impl< )?; ASM::ret(&mut out); - // Update relocs to include stack setup offset. + // Update other relocs to include stack setup offset. let mut out_relocs = bumpalo::vec![in self.env.arena]; - let old_relocs = std::mem::replace(&mut self.relocs, bumpalo::vec![in self.env.arena]); - out_relocs.extend(old_relocs.into_iter().map(|reloc| match reloc { - Relocation::LocalData { offset, data } => Relocation::LocalData { - offset: offset + setup_offset as u64, - data, - }, - Relocation::LinkedData { offset, name } => Relocation::LinkedData { - offset: offset + setup_offset as u64, - name, - }, - Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { - offset: offset + setup_offset as u64, - name, - }, - })); + out_relocs.extend( + old_relocs + .into_iter() + .filter(|reloc| !matches!(reloc, Relocation::JmpToReturn { .. })) + .map(|reloc| match reloc { + Relocation::LocalData { offset, data } => Relocation::LocalData { + offset: offset + setup_offset as u64, + data, + }, + Relocation::LinkedData { offset, name } => Relocation::LinkedData { + offset: offset + setup_offset as u64, + name, + }, + Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { + offset: offset + setup_offset as u64, + name, + }, + Relocation::JmpToReturn { .. } => unreachable!(), + }), + ); Ok((out.into_bump_slice(), out_relocs.into_bump_slice())) } @@ -401,29 +456,13 @@ impl< arg_layouts: &[Layout<'a>], ret_layout: &Layout<'a>, ) -> Result<(), String> { + if let Some(SelfRecursive::SelfRecursive(id)) = self.is_self_recursive { + if &fn_name == self.proc_name.as_ref().unwrap() && self.join_map.contains_key(&id) { + return self.build_jump(&id, args, arg_layouts, ret_layout); + } + } // Save used caller saved regs. - let old_general_used_regs = std::mem::replace( - &mut self.general_used_regs, - bumpalo::vec![in self.env.arena], - ); - for (reg, saved_sym) in old_general_used_regs.into_iter() { - if CC::general_caller_saved(®) { - self.general_free_regs.push(reg); - self.free_to_stack(&saved_sym)?; - } else { - self.general_used_regs.push((reg, saved_sym)); - } - } - let old_float_used_regs = - std::mem::replace(&mut self.float_used_regs, bumpalo::vec![in self.env.arena]); - for (reg, saved_sym) in old_float_used_regs.into_iter() { - if CC::float_caller_saved(®) { - self.float_free_regs.push(reg); - self.free_to_stack(&saved_sym)?; - } else { - self.float_used_regs.push((reg, saved_sym)); - } - } + self.push_used_caller_saved_regs_to_stack()?; // Put values in param regs or on top of the stack. let tmp_stack_size = CC::store_args( @@ -486,7 +525,7 @@ impl< // Build unconditional jump to the end of this switch. // Since we don't know the offset yet, set it to 0 and overwrite later. let jmp_location = self.buf.len(); - let jmp_offset = ASM::jmp_imm32(&mut self.buf, 0); + let jmp_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); ret_jumps.push((jmp_location, jmp_offset)); // Overwite the original jne with the correct offset. @@ -510,12 +549,12 @@ impl< // Update all return jumps to jump past the default case. let ret_offset = self.buf.len(); for (jmp_location, start_offset) in ret_jumps.into_iter() { - tmp.clear(); - let jmp_offset = ret_offset - start_offset; - ASM::jmp_imm32(&mut tmp, jmp_offset as i32); - for (i, byte) in tmp.iter().enumerate() { - self.buf[jmp_location + i] = *byte; - } + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + ret_offset as u64, + ); } Ok(()) } else { @@ -526,6 +565,134 @@ impl< } } + fn build_join( + &mut self, + id: &JoinPointId, + parameters: &'a [Param<'a>], + body: &'a Stmt<'a>, + remainder: &'a Stmt<'a>, + ret_layout: &Layout<'a>, + ) -> Result<(), String> { + // Create jump to remaining. + let jmp_location = self.buf.len(); + let start_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); + + // This section can essentially be seen as a sub function within the main function. + // Thus we build using a new backend with some minor extra synchronization. + let mut sub_backend = Self::new(self.env)?; + sub_backend.reset( + self.proc_name.as_ref().unwrap().clone(), + self.is_self_recursive.as_ref().unwrap().clone(), + ); + // Sync static maps of important information. + sub_backend.last_seen_map = self.last_seen_map.clone(); + sub_backend.layout_map = self.layout_map.clone(); + sub_backend.free_map = self.free_map.clone(); + + // Setup join point. + sub_backend.join_map.insert(*id, 0); + self.join_map.insert(*id, self.buf.len() as u64); + + // Sync stack size so the "sub function" doesn't mess up our stack. + sub_backend.stack_size = self.stack_size; + sub_backend.fn_call_stack_size = self.fn_call_stack_size; + + // Load params as if they were args. + let mut args = bumpalo::vec![in self.env.arena]; + for param in parameters { + args.push((param.layout, param.symbol)); + } + sub_backend.load_args(args.into_bump_slice(), ret_layout)?; + + // Build all statements in body. + sub_backend.build_stmt(body, ret_layout)?; + + // Merge the "sub function" into the main function. + let sub_func_offset = self.buf.len() as u64; + self.buf.extend_from_slice(&sub_backend.buf); + // Update stack based on how much was used by the sub function. + self.stack_size = sub_backend.stack_size; + self.fn_call_stack_size = sub_backend.fn_call_stack_size; + // Relocations must be shifted to be merged correctly. + self.relocs + .extend(sub_backend.relocs.into_iter().map(|reloc| match reloc { + Relocation::LocalData { offset, data } => Relocation::LocalData { + offset: offset + sub_func_offset, + data, + }, + Relocation::LinkedData { offset, name } => Relocation::LinkedData { + offset: offset + sub_func_offset, + name, + }, + Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { + offset: offset + sub_func_offset, + name, + }, + Relocation::JmpToReturn { + inst_loc, + inst_size, + offset, + } => Relocation::JmpToReturn { + inst_loc: inst_loc + sub_func_offset, + inst_size, + offset: offset + sub_func_offset, + }, + })); + + // Overwite the original jump with the correct offset. + let mut tmp = bumpalo::vec![in self.env.arena]; + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + self.buf.len() as u64, + ); + + // Build remainder of function. + self.build_stmt(remainder, ret_layout) + } + + fn build_jump( + &mut self, + id: &JoinPointId, + args: &'a [Symbol], + arg_layouts: &[Layout<'a>], + ret_layout: &Layout<'a>, + ) -> Result<(), String> { + // Treat this like a function call, but with a jump instead of a call instruction at the end. + + self.push_used_caller_saved_regs_to_stack()?; + + let tmp_stack_size = CC::store_args( + &mut self.buf, + &self.symbol_storage_map, + args, + arg_layouts, + ret_layout, + )?; + self.fn_call_stack_size = std::cmp::max(self.fn_call_stack_size, tmp_stack_size); + + let jmp_location = self.buf.len(); + let start_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); + + if let Some(offset) = self.join_map.get(id) { + let offset = *offset; + let mut tmp = bumpalo::vec![in self.env.arena]; + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + offset, + ); + Ok(()) + } else { + Err(format!( + "Jump: unknown point specified to jump to: {:?}", + id + )) + } + } + fn build_num_abs( &mut self, dst: &Symbol, @@ -828,29 +995,26 @@ impl< fn return_symbol(&mut self, sym: &Symbol, layout: &Layout<'a>) -> Result<(), String> { let val = self.symbol_storage_map.get(sym); match val { - Some(SymbolStorage::GeneralReg(reg)) if *reg == CC::GENERAL_RETURN_REGS[0] => Ok(()), + Some(SymbolStorage::GeneralReg(reg)) if *reg == CC::GENERAL_RETURN_REGS[0] => {} Some(SymbolStorage::GeneralReg(reg)) => { // If it fits in a general purpose register, just copy it over to. // Technically this can be optimized to produce shorter instructions if less than 64bits. ASM::mov_reg64_reg64(&mut self.buf, CC::GENERAL_RETURN_REGS[0], *reg); - Ok(()) } - Some(SymbolStorage::FloatReg(reg)) if *reg == CC::FLOAT_RETURN_REGS[0] => Ok(()), + Some(SymbolStorage::FloatReg(reg)) if *reg == CC::FLOAT_RETURN_REGS[0] => {} Some(SymbolStorage::FloatReg(reg)) => { ASM::mov_freg64_freg64(&mut self.buf, CC::FLOAT_RETURN_REGS[0], *reg); - Ok(()) } Some(SymbolStorage::Base { offset, size, .. }) => match layout { Layout::Builtin(Builtin::Int64) => { ASM::mov_reg64_base32(&mut self.buf, CC::GENERAL_RETURN_REGS[0], *offset); - Ok(()) } Layout::Builtin(Builtin::Float64) => { ASM::mov_freg64_base32(&mut self.buf, CC::FLOAT_RETURN_REGS[0], *offset); - Ok(()) } Layout::Struct(field_layouts) => { let (offset, size) = (*offset, *size); + // Nothing to do for empty struct if size > 0 { let ret_reg = if self.symbol_storage_map.contains_key(&Symbol::RET_POINTER) { @@ -858,23 +1022,34 @@ impl< } else { None }; - CC::return_struct(&mut self.buf, offset, size, field_layouts, ret_reg) - } else { - // Nothing to do for empty struct - Ok(()) + CC::return_struct(&mut self.buf, offset, size, field_layouts, ret_reg)?; } } - x => Err(format!( - "returning symbol with layout, {:?}, is not yet implemented", - x - )), + x => { + return Err(format!( + "returning symbol with layout, {:?}, is not yet implemented", + x + )); + } }, - Some(x) => Err(format!( - "returning symbol storage, {:?}, is not yet implemented", - x - )), - None => Err(format!("Unknown return symbol: {}", sym)), + Some(x) => { + return Err(format!( + "returning symbol storage, {:?}, is not yet implemented", + x + )); + } + None => { + return Err(format!("Unknown return symbol: {}", sym)); + } } + let inst_loc = self.buf.len() as u64; + let offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678) as u64; + self.relocs.push(Relocation::JmpToReturn { + inst_loc, + inst_size: self.buf.len() as u64 - inst_loc, + offset, + }); + Ok(()) } } @@ -1212,4 +1387,74 @@ impl< )), } } + + fn push_used_caller_saved_regs_to_stack(&mut self) -> Result<(), String> { + let old_general_used_regs = std::mem::replace( + &mut self.general_used_regs, + bumpalo::vec![in self.env.arena], + ); + for (reg, saved_sym) in old_general_used_regs.into_iter() { + if CC::general_caller_saved(®) { + self.general_free_regs.push(reg); + self.free_to_stack(&saved_sym)?; + } else { + self.general_used_regs.push((reg, saved_sym)); + } + } + let old_float_used_regs = + std::mem::replace(&mut self.float_used_regs, bumpalo::vec![in self.env.arena]); + for (reg, saved_sym) in old_float_used_regs.into_iter() { + if CC::float_caller_saved(®) { + self.float_free_regs.push(reg); + self.free_to_stack(&saved_sym)?; + } else { + self.float_used_regs.push((reg, saved_sym)); + } + } + Ok(()) + } + + // Updates a jump instruction to a new offset and returns the number of bytes written. + fn update_jmp_imm32_offset( + &mut self, + tmp: &mut Vec<'a, u8>, + jmp_location: u64, + base_offset: u64, + target_offset: u64, + ) { + tmp.clear(); + let jmp_offset = target_offset as i32 - base_offset as i32; + ASM::jmp_imm32(tmp, jmp_offset); + for (i, byte) in tmp.iter().enumerate() { + self.buf[jmp_location as usize + i] = *byte; + } + } +} + +#[macro_export] +macro_rules! single_register_integers { + () => { + Builtin::Int1 + | Builtin::Int8 + | Builtin::Int16 + | Builtin::Int32 + | Builtin::Int64 + | Builtin::Usize + }; +} + +#[macro_export] +macro_rules! single_register_floats { + () => { + // Float16 is explicitly ignored because it is not supported by must hardware and may require special exceptions. + // Builtin::Float16 | + Builtin::Float32 | Builtin::Float64 + }; +} + +#[macro_export] +macro_rules! single_register_builtins { + () => { + single_register_integers!() | single_register_floats!() + }; } diff --git a/compiler/gen_dev/src/generic64/x86_64.rs b/compiler/gen_dev/src/generic64/x86_64.rs index d052c44e43..a39a828f8a 100644 --- a/compiler/gen_dev/src/generic64/x86_64.rs +++ b/compiler/gen_dev/src/generic64/x86_64.rs @@ -1,5 +1,7 @@ use crate::generic64::{Assembler, CallConv, RegTrait, SymbolStorage, PTR_SIZE}; -use crate::Relocation; +use crate::{ + single_register_builtins, single_register_floats, single_register_integers, Relocation, +}; use bumpalo::collections::Vec; use roc_collections::all::MutMap; use roc_module::symbol::Symbol; @@ -191,7 +193,7 @@ impl CallConv for X86_64SystemV { } for (layout, sym) in args.iter() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if general_i < Self::GENERAL_PARAM_REGS.len() { symbol_map.insert( *sym, @@ -210,7 +212,7 @@ impl CallConv for X86_64SystemV { ); } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if float_i < Self::FLOAT_PARAM_REGS.len() { symbol_map.insert( *sym, @@ -229,6 +231,7 @@ impl CallConv for X86_64SystemV { ); } } + Layout::Struct(&[]) => {} x => { return Err(format!( "Loading args with layout {:?} not yet implementd", @@ -254,8 +257,7 @@ impl CallConv for X86_64SystemV { // For most return layouts we will do nothing. // In some cases, we need to put the return address as the first arg. match ret_layout { - Layout::Builtin(Builtin::Int64) => {} - Layout::Builtin(Builtin::Float64) => {} + Layout::Builtin(single_register_builtins!()) => {} x => { return Err(format!( "receiving return type, {:?}, is not yet implemented", @@ -265,7 +267,7 @@ impl CallConv for X86_64SystemV { } for (i, layout) in arg_layouts.iter().enumerate() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if general_i < Self::GENERAL_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::GENERAL_PARAM_REGS[general_i]; @@ -319,7 +321,7 @@ impl CallConv for X86_64SystemV { stack_offset += 8; } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if float_i < Self::FLOAT_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::FLOAT_PARAM_REGS[float_i]; @@ -371,6 +373,7 @@ impl CallConv for X86_64SystemV { stack_offset += 8; } } + Layout::Struct(&[]) => {} x => { return Err(format!( "calling with arg type, {:?}, is not yet implemented", @@ -529,13 +532,14 @@ impl CallConv for X86_64WindowsFastcall { for (layout, sym) in args.iter() { if i < Self::GENERAL_PARAM_REGS.len() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { symbol_map .insert(*sym, SymbolStorage::GeneralReg(Self::GENERAL_PARAM_REGS[i])); } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { symbol_map.insert(*sym, SymbolStorage::FloatReg(Self::FLOAT_PARAM_REGS[i])); } + Layout::Struct(&[]) => {} x => { return Err(format!( "Loading args with layout {:?} not yet implementd", @@ -546,8 +550,7 @@ impl CallConv for X86_64WindowsFastcall { i += 1; } else { base_offset += match layout { - Layout::Builtin(Builtin::Int64) => 8, - Layout::Builtin(Builtin::Float64) => 8, + Layout::Builtin(single_register_builtins!()) => 8, x => { return Err(format!( "Loading args with layout {:?} not yet implemented", @@ -581,8 +584,7 @@ impl CallConv for X86_64WindowsFastcall { // For most return layouts we will do nothing. // In some cases, we need to put the return address as the first arg. match ret_layout { - Layout::Builtin(Builtin::Int64) => {} - Layout::Builtin(Builtin::Float64) => {} + Layout::Builtin(single_register_builtins!()) => {} x => { return Err(format!( "receiving return type, {:?}, is not yet implemented", @@ -592,7 +594,7 @@ impl CallConv for X86_64WindowsFastcall { } for (i, layout) in arg_layouts.iter().enumerate() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if i < Self::GENERAL_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::GENERAL_PARAM_REGS[reg_i]; @@ -646,7 +648,7 @@ impl CallConv for X86_64WindowsFastcall { stack_offset += 8; } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if i < Self::FLOAT_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::FLOAT_PARAM_REGS[reg_i]; @@ -698,6 +700,7 @@ impl CallConv for X86_64WindowsFastcall { stack_offset += 8; } } + Layout::Struct(&[]) => {} x => { return Err(format!( "calling with arg type, {:?}, is not yet implemented", diff --git a/compiler/gen_dev/src/lib.rs b/compiler/gen_dev/src/lib.rs index 70e9bf0f61..010fa066d3 100644 --- a/compiler/gen_dev/src/lib.rs +++ b/compiler/gen_dev/src/lib.rs @@ -9,10 +9,10 @@ use roc_module::ident::{ModuleName, TagName}; use roc_module::low_level::LowLevel; use roc_module::symbol::{Interns, Symbol}; use roc_mono::ir::{ - BranchInfo, CallType, Expr, JoinPointId, ListLiteralElement, Literal, Proc, Stmt, + BranchInfo, CallType, Expr, JoinPointId, ListLiteralElement, Literal, Param, Proc, + SelfRecursive, Stmt, }; use roc_mono::layout::{Builtin, Layout, LayoutIds}; -use target_lexicon::Triple; mod generic64; mod object_builder; @@ -46,6 +46,11 @@ pub enum Relocation { offset: u64, name: String, }, + JmpToReturn { + inst_loc: u64, + inst_size: u64, + offset: u64, + }, } trait Backend<'a> @@ -53,12 +58,13 @@ where Self: Sized, { /// new creates a new backend that will output to the specific Object. - fn new(env: &'a Env, target: &Triple) -> Result; + fn new(env: &'a Env) -> Result; fn env(&self) -> &'a Env<'a>; /// reset resets any registers or other values that may be occupied at the end of a procedure. - fn reset(&mut self); + /// It also passes basic procedure information to the builder for setup of the next function. + fn reset(&mut self, name: String, is_self_recursive: SelfRecursive); /// finalize does any setup and cleanup that should happen around the procedure. /// finalize does setup because things like stack size and jump locations are not know until the function is written. @@ -79,7 +85,10 @@ where /// build_proc creates a procedure and outputs it to the wrapped object writer. fn build_proc(&mut self, proc: Proc<'a>) -> Result<(&'a [u8], &[Relocation]), String> { - self.reset(); + let proc_name = LayoutIds::default() + .get(proc.name, &proc.ret_layout) + .to_symbol_string(proc.name, &self.env().interns); + self.reset(proc_name, proc.is_self_recursive); self.load_args(proc.args, &proc.ret_layout)?; for (layout, sym) in proc.args { self.set_layout_map(*sym, layout)?; @@ -128,6 +137,35 @@ where self.free_symbols(stmt)?; Ok(()) } + Stmt::Join { + id, + parameters, + body, + remainder, + } => { + for param in parameters.iter() { + self.set_layout_map(param.symbol, ¶m.layout)?; + } + self.build_join(id, parameters, body, remainder, ret_layout)?; + self.free_symbols(stmt)?; + Ok(()) + } + Stmt::Jump(id, args) => { + let mut arg_layouts: bumpalo::collections::Vec> = + bumpalo::vec![in self.env().arena]; + arg_layouts.reserve(args.len()); + let layout_map = self.layout_map(); + for arg in *args { + if let Some(layout) = layout_map.get(arg) { + arg_layouts.push(*layout); + } else { + return Err(format!("the argument, {:?}, has no know layout", arg)); + } + } + self.build_jump(id, args, arg_layouts.into_bump_slice(), ret_layout)?; + self.free_symbols(stmt)?; + Ok(()) + } x => Err(format!("the statement, {:?}, is not yet implemented", x)), } } @@ -141,6 +179,25 @@ where ret_layout: &Layout<'a>, ) -> Result<(), String>; + // build_join generates a instructions for a join statement. + fn build_join( + &mut self, + id: &JoinPointId, + parameters: &'a [Param<'a>], + body: &'a Stmt<'a>, + remainder: &'a Stmt<'a>, + ret_layout: &Layout<'a>, + ) -> Result<(), String>; + + // build_jump generates a instructions for a jump statement. + fn build_jump( + &mut self, + id: &JoinPointId, + args: &'a [Symbol], + arg_layouts: &[Layout<'a>], + ret_layout: &Layout<'a>, + ) -> Result<(), String>; + /// build_expr builds the expressions for the specified symbol. /// The builder must keep track of the symbol because it may be referred to later. fn build_expr( @@ -263,8 +320,7 @@ where let layout_map = self.layout_map(); for arg in *arguments { if let Some(layout) = layout_map.get(arg) { - // This is safe because every value in the map is always set with a valid layout and cannot be null. - arg_layouts.push(unsafe { *(*layout) }); + arg_layouts.push(*layout); } else { return Err(format!("the argument, {:?}, has no know layout", arg)); } @@ -507,7 +563,7 @@ where /// load_literal sets a symbol to be equal to a literal. fn load_literal(&mut self, sym: &Symbol, lit: &Literal<'a>) -> Result<(), String>; - /// return_symbol moves a symbol to the correct return location for the backend. + /// return_symbol moves a symbol to the correct return location for the backend and adds a jump to the end of the function. fn return_symbol(&mut self, sym: &Symbol, layout: &Layout<'a>) -> Result<(), String>; /// free_symbols will free all symbols for the given statement. @@ -542,12 +598,10 @@ where /// set_layout_map sets the layout for a specific symbol. fn set_layout_map(&mut self, sym: Symbol, layout: &Layout<'a>) -> Result<(), String> { - if let Some(x) = self.layout_map().insert(sym, layout) { + if let Some(old_layout) = self.layout_map().insert(sym, *layout) { // Layout map already contains the symbol. We should never need to overwrite. // If the layout is not the same, that is a bug. - // There is always an old layout value and this dereference is safe. - let old_layout = unsafe { *x }; - if old_layout != *layout { + if &old_layout != layout { Err(format!( "Overwriting layout for symbol, {:?}. This should never happen. got {:?}, want {:?}", sym, layout, old_layout @@ -561,7 +615,7 @@ where } /// layout_map gets the map from symbol to layout. - fn layout_map(&mut self) -> &mut MutMap>; + fn layout_map(&mut self) -> &mut MutMap>; fn create_free_map(&mut self) { let mut free_map = MutMap::default(); diff --git a/compiler/gen_dev/src/object_builder.rs b/compiler/gen_dev/src/object_builder.rs index 52ae6b4b08..9e5ae6f76e 100644 --- a/compiler/gen_dev/src/object_builder.rs +++ b/compiler/gen_dev/src/object_builder.rs @@ -34,7 +34,7 @@ pub fn build_module<'a>( x86_64::X86_64FloatReg, x86_64::X86_64Assembler, x86_64::X86_64SystemV, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -52,7 +52,7 @@ pub fn build_module<'a>( x86_64::X86_64FloatReg, x86_64::X86_64Assembler, x86_64::X86_64SystemV, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -74,7 +74,7 @@ pub fn build_module<'a>( aarch64::AArch64FloatReg, aarch64::AArch64Assembler, aarch64::AArch64Call, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -191,10 +191,16 @@ fn build_object<'a, B: Backend<'a>>( let mut layout_ids = roc_mono::layout::LayoutIds::default(); let mut procs = Vec::with_capacity_in(procedures.len(), env.arena); for ((sym, layout), proc) in procedures { - let fn_name = layout_ids + let base_name = layout_ids .get_toplevel(sym, &layout) .to_symbol_string(sym, &env.interns); + let fn_name = if env.exposed_to_host.contains(&sym) { + format!("roc_{}_exposed", base_name) + } else { + base_name + }; + let section_id = output.add_section( output.segment_name(StandardSegment::Text).to_vec(), format!(".text.{:x}", sym.as_u64()).as_bytes().to_vec(), @@ -298,6 +304,7 @@ fn build_object<'a, B: Backend<'a>>( return Err(format!("failed to find fn symbol for {:?}", name)); } } + Relocation::JmpToReturn { .. } => unreachable!(), }; relocations.push((section_id, elfreloc)); } diff --git a/compiler/gen_dev/tests/dev_num.rs b/compiler/gen_dev/tests/dev_num.rs index d1c0ae0d75..c223acbcdf 100644 --- a/compiler/gen_dev/tests/dev_num.rs +++ b/compiler/gen_dev/tests/dev_num.rs @@ -281,6 +281,24 @@ mod dev_num { ); } + #[test] + fn gen_fast_fib_fn() { + assert_evals_to!( + indoc!( + r#" + fib = \n, a, b -> + if n == 0 then + a + else + fib (n - 1) b (a + b) + fib 10 0 1 + "# + ), + 55, + i64 + ); + } + #[test] fn f64_abs() { assert_evals_to!("Num.abs -4.7", 4.7, f64); @@ -580,18 +598,18 @@ mod dev_num { // assert_evals_to!("0.0 >= 0.0", true, bool); // } - // #[test] - // fn gen_order_of_arithmetic_ops() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 + 3 * 7 - 2 - // "# - // ), - // 20, - // i64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops() { + assert_evals_to!( + indoc!( + r#" + 1 + 3 * 7 - 2 + "# + ), + 20, + i64 + ); + } // #[test] // fn gen_order_of_arithmetic_ops_complex_float() { @@ -606,59 +624,59 @@ mod dev_num { // ); // } - // #[test] - // fn if_guard_bind_variable_false() { - // assert_evals_to!( - // indoc!( - // r#" - // wrapper = \{} -> - // when 10 is - // x if x == 5 -> 0 - // _ -> 42 + #[test] + fn if_guard_bind_variable_false() { + assert_evals_to!( + indoc!( + r#" + wrapper = \{} -> + when 10 is + x if x == 5 -> 0 + _ -> 42 - // wrapper {} - // "# - // ), - // 42, - // i64 - // ); - // } + wrapper {} + "# + ), + 42, + i64 + ); + } - // #[test] - // fn if_guard_bind_variable_true() { - // assert_evals_to!( - // indoc!( - // r#" - // wrapper = \{} -> - // when 10 is - // x if x == 10 -> 42 - // _ -> 0 + #[test] + fn if_guard_bind_variable_true() { + assert_evals_to!( + indoc!( + r#" + wrapper = \{} -> + when 10 is + x if x == 10 -> 42 + _ -> 0 - // wrapper {} - // "# - // ), - // 42, - // i64 - // ); - // } + wrapper {} + "# + ), + 42, + i64 + ); + } - // #[test] - // fn tail_call_elimination() { - // assert_evals_to!( - // indoc!( - // r#" - // sum = \n, accum -> - // when n is - // 0 -> accum - // _ -> sum (n - 1) (n + accum) + #[test] + fn tail_call_elimination() { + assert_evals_to!( + indoc!( + r#" + sum = \n, accum -> + when n is + 0 -> accum + _ -> sum (n - 1) (n + accum) - // sum 1_000_000 0 - // "# - // ), - // 500000500000, - // i64 - // ); - // } + sum 1_000_000 0 + "# + ), + 500000500000, + i64 + ); + } // #[test] // fn int_negate() { diff --git a/compiler/gen_dev/tests/helpers/eval.rs b/compiler/gen_dev/tests/helpers/eval.rs index 675f981e72..84e5dfdf22 100644 --- a/compiler/gen_dev/tests/helpers/eval.rs +++ b/compiler/gen_dev/tests/helpers/eval.rs @@ -94,10 +94,12 @@ pub fn helper<'a>( let main_fn_layout = loaded.entry_point.layout; let mut layout_ids = roc_mono::layout::LayoutIds::default(); - let main_fn_name = layout_ids + let main_fn_name_base = layout_ids .get_toplevel(main_fn_symbol, &main_fn_layout) .to_symbol_string(main_fn_symbol, &interns); + let main_fn_name = format!("roc_{}_exposed", main_fn_name_base); + let mut lines = Vec::new(); // errors whose reporting we delay (so we can see that code gen generates runtime errors) let mut delayed_errors = Vec::new(); @@ -143,12 +145,13 @@ pub fn helper<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/compiler/gen_llvm/src/llvm/bitcode.rs b/compiler/gen_llvm/src/llvm/bitcode.rs index f873af0518..52b604dff3 100644 --- a/compiler/gen_llvm/src/llvm/bitcode.rs +++ b/compiler/gen_llvm/src/llvm/bitcode.rs @@ -10,7 +10,7 @@ use inkwell::types::{BasicType, BasicTypeEnum}; use inkwell::values::{BasicValue, BasicValueEnum, CallSiteValue, FunctionValue, InstructionValue}; use inkwell::AddressSpace; use roc_module::symbol::Symbol; -use roc_mono::layout::{Layout, LayoutIds, UnionLayout}; +use roc_mono::layout::{LambdaSet, Layout, LayoutIds, UnionLayout}; pub fn call_bitcode_fn<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, @@ -189,7 +189,7 @@ fn build_has_tag_id_help<'a, 'ctx, 'env>( pub fn build_transform_caller<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, argument_layouts: &[Layout<'a>], ) -> FunctionValue<'ctx> { let fn_name: &str = &format!( @@ -212,7 +212,7 @@ pub fn build_transform_caller<'a, 'ctx, 'env>( fn build_transform_caller_help<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, roc_function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, argument_layouts: &[Layout<'a>], fn_name: &str, ) -> FunctionValue<'ctx> { @@ -270,7 +270,7 @@ fn build_transform_caller_help<'a, 'ctx, 'env>( arguments_cast.push(argument); } - match closure_data_layout { + match closure_data_layout.runtime_representation() { Layout::Struct(&[]) => { // nothing to add } @@ -529,7 +529,7 @@ pub fn build_eq_wrapper<'a, 'ctx, 'env>( pub fn build_compare_wrapper<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, roc_function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, layout: &Layout<'a>, ) -> FunctionValue<'ctx> { let block = env.builder.get_insert_block().expect("to be in a function"); @@ -595,7 +595,7 @@ pub fn build_compare_wrapper<'a, 'ctx, 'env>( let default = [value1, value2]; - let arguments_cast = match closure_data_layout { + let arguments_cast = match closure_data_layout.runtime_representation() { Layout::Struct(&[]) => { // nothing to add &default diff --git a/compiler/gen_llvm/src/llvm/build.rs b/compiler/gen_llvm/src/llvm/build.rs index b8c9b922e8..4864b04fcb 100644 --- a/compiler/gen_llvm/src/llvm/build.rs +++ b/compiler/gen_llvm/src/llvm/build.rs @@ -3,15 +3,15 @@ use std::path::Path; use crate::llvm::bitcode::{call_bitcode_fn, call_void_bitcode_fn}; use crate::llvm::build_dict::{ - dict_contains, dict_difference, dict_empty, dict_get, dict_insert, dict_intersection, + self, dict_contains, dict_difference, dict_empty, dict_get, dict_insert, dict_intersection, dict_keys, dict_len, dict_remove, dict_union, dict_values, dict_walk, set_from_list, }; use crate::llvm::build_hash::generic_hash; use crate::llvm::build_list::{ - allocate_list, empty_list, empty_polymorphic_list, list_append, list_concat, list_contains, - list_drop, list_get_unsafe, list_join, list_keep_errs, list_keep_if, list_keep_oks, list_len, - list_map, list_map2, list_map3, list_map_with_index, list_prepend, list_range, list_repeat, - list_reverse, list_set, list_single, list_sort_with, list_swap, + self, allocate_list, empty_list, empty_polymorphic_list, list_append, list_concat, + list_contains, list_drop, list_get_unsafe, list_join, list_keep_errs, list_keep_if, + list_keep_oks, list_len, list_map, list_map2, list_map3, list_map_with_index, list_prepend, + list_range, list_repeat, list_reverse, list_set, list_single, list_sort_with, list_swap, }; use crate::llvm::build_str::{ empty_str, str_concat, str_count_graphemes, str_ends_with, str_from_float, str_from_int, @@ -638,7 +638,7 @@ pub fn construct_optimization_passes<'a>( let pmb = PassManagerBuilder::create(); match opt_level { - OptLevel::Normal => { + OptLevel::Development | OptLevel::Normal => { pmb.set_optimization_level(OptimizationLevel::None); } OptLevel::Optimize => { @@ -704,7 +704,14 @@ fn promote_to_main_function<'a, 'ctx, 'env>( let main_fn_name = "$Test.main"; // Add main to the module. - let main_fn = expose_function_to_host_help(env, main_fn_name, roc_main_fn, main_fn_name); + let main_fn = expose_function_to_host_help_c_abi( + env, + main_fn_name, + roc_main_fn, + &[], + top_level.result, + main_fn_name, + ); (main_fn_name, main_fn) } @@ -1163,8 +1170,16 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( StructAtIndex { index, structure, .. } => { + let (value, layout) = load_symbol_and_layout(scope, structure); + + let layout = if let Layout::LambdaSet(lambda_set) = layout { + lambda_set.runtime_representation() + } else { + *layout + }; + // extract field from a record - match load_symbol_and_layout(scope, structure) { + match (value, layout) { (StructValue(argument), Layout::Struct(fields)) => { debug_assert!(!fields.is_empty()); env.builder @@ -2304,32 +2319,6 @@ fn list_literal<'a, 'ctx, 'env>( } } -fn decrement_with_size_check<'a, 'ctx, 'env>( - env: &Env<'a, 'ctx, 'env>, - parent: FunctionValue<'ctx>, - size: IntValue<'ctx>, - layout: Layout<'a>, - refcount_ptr: PointerToRefcount<'ctx>, -) { - let not_empty = env.context.append_basic_block(parent, "not_null"); - - let done = env.context.append_basic_block(parent, "done"); - - let is_empty = - env.builder - .build_int_compare(IntPredicate::EQ, size, size.get_type().const_zero(), ""); - - env.builder - .build_conditional_branch(is_empty, done, not_empty); - - env.builder.position_at_end(not_empty); - - refcount_ptr.decrement(env, &layout); - - env.builder.build_unconditional_branch(done); - env.builder.position_at_end(done); -} - pub fn build_exp_stmt<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, layout_ids: &mut LayoutIds<'a>, @@ -2539,34 +2528,25 @@ pub fn build_exp_stmt<'a, 'ctx, 'env>( let (value, layout) = load_symbol_and_layout(scope, symbol); match layout { - Layout::Builtin(Builtin::List(_)) => { + Layout::Builtin(Builtin::List(element_layout)) => { debug_assert!(value.is_struct_value()); + let alignment = element_layout.alignment_bytes(env.ptr_bytes); - // because of how we insert DECREF for lists, we can't guarantee that - // the list is non-empty. When the list is empty, the pointer to the - // elements is NULL, and trying to get to the RC address will - // underflow, causing a segfault. Therefore, in this case we must - // manually check that the list is non-empty - let refcount_ptr = PointerToRefcount::from_list_wrapper( - env, - value.into_struct_value(), - ); - - let length = list_len(env.builder, value.into_struct_value()); - - decrement_with_size_check(env, parent, length, *layout, refcount_ptr); + build_list::decref(env, value.into_struct_value(), alignment); } - Layout::Builtin(Builtin::Dict(_, _)) | Layout::Builtin(Builtin::Set(_)) => { + Layout::Builtin(Builtin::Dict(key_layout, value_layout)) => { debug_assert!(value.is_struct_value()); + let alignment = key_layout + .alignment_bytes(env.ptr_bytes) + .max(value_layout.alignment_bytes(env.ptr_bytes)); - let refcount_ptr = PointerToRefcount::from_list_wrapper( - env, - value.into_struct_value(), - ); + build_dict::decref(env, value.into_struct_value(), alignment); + } + Layout::Builtin(Builtin::Set(key_layout)) => { + debug_assert!(value.is_struct_value()); + let alignment = key_layout.alignment_bytes(env.ptr_bytes); - let length = dict_len(env, scope, *symbol).into_int_value(); - - decrement_with_size_check(env, parent, length, *layout, refcount_ptr); + build_dict::decref(env, value.into_struct_value(), alignment); } _ if layout.is_refcounted() => { @@ -2635,6 +2615,18 @@ pub fn load_symbol_and_layout<'a, 'ctx, 'b>( None => panic!("There was no entry for {:?} in scope {:?}", symbol, scope), } } + +pub fn load_symbol_and_lambda_set<'a, 'ctx, 'b>( + scope: &'b Scope<'a, 'ctx>, + symbol: &Symbol, +) -> (BasicValueEnum<'ctx>, LambdaSet<'a>) { + match scope.get(symbol) { + Some((Layout::LambdaSet(lambda_set), ptr)) => (*ptr, *lambda_set), + Some((other, ptr)) => panic!("Not a lambda set: {:?}, {:?}", other, ptr), + None => panic!("There was no entry for {:?} in scope {:?}", symbol, scope), + } +} + fn access_index_struct_value<'ctx>( builder: &Builder<'ctx>, from_value: StructValue<'ctx>, @@ -3088,33 +3080,42 @@ fn expose_function_to_host<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, symbol: Symbol, roc_function: FunctionValue<'ctx>, + arguments: &[Layout<'a>], + return_layout: Layout<'a>, ) { // Assumption: there is only one specialization of a host-exposed function let ident_string = symbol.as_str(&env.interns); let c_function_name: String = format!("roc__{}_1_exposed", ident_string); - expose_function_to_host_help(env, ident_string, roc_function, &c_function_name); + expose_function_to_host_help_c_abi( + env, + ident_string, + roc_function, + arguments, + return_layout, + &c_function_name, + ); } -fn expose_function_to_host_help<'a, 'ctx, 'env>( +fn expose_function_to_host_help_c_abi_generic<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, - ident_string: &str, roc_function: FunctionValue<'ctx>, + arguments: &[Layout<'a>], c_function_name: &str, ) -> FunctionValue<'ctx> { - let context = env.context; + // NOTE we ingore env.is_gen_test here + let wrapper_return_type = roc_function.get_type().get_return_type().unwrap(); - let wrapper_return_type = context.struct_type( - &[ - context.i64_type().into(), - roc_function.get_type().get_return_type().unwrap(), - ], - false, - ); + let mut cc_argument_types = Vec::with_capacity_in(arguments.len(), env.arena); + for layout in arguments { + cc_argument_types.push(to_cc_type(env, layout)); + } // STEP 1: turn `f : a,b,c -> d` into `f : a,b,c, &d -> {}` - let mut argument_types = roc_function.get_type().get_param_types(); + // let mut argument_types = roc_function.get_type().get_param_types(); + let mut argument_types = cc_argument_types; let return_type = wrapper_return_type; + let output_type = return_type.ptr_type(AddressSpace::Generic); argument_types.push(output_type.into()); @@ -3142,43 +3143,236 @@ fn expose_function_to_host_help<'a, 'ctx, 'env>( debug_info_init!(env, c_function); // drop the final argument, which is the pointer we write the result into - let args = c_function.get_params(); - let output_arg_index = args.len() - 1; - let args = &args[..args.len() - 1]; + let args_vector = c_function.get_params(); + let mut args = args_vector.as_slice(); + let args_length = args.len(); + + args = &args[..args.len() - 1]; + + let mut arguments_for_call = Vec::with_capacity_in(args.len(), env.arena); + + let it = args.iter().zip(roc_function.get_type().get_param_types()); + for (arg, fastcc_type) in it { + let arg_type = arg.get_type(); + if arg_type == fastcc_type { + // the C and Fast calling conventions agree + arguments_for_call.push(*arg); + } else { + let cast = complex_bitcast_check_size(env, *arg, fastcc_type, "to_fastcc_type"); + arguments_for_call.push(cast); + } + } + + let arguments_for_call = &arguments_for_call.into_bump_slice(); debug_assert_eq!(args.len(), roc_function.get_params().len()); let call_result = { if env.is_gen_test { let roc_wrapper_function = make_exception_catcher(env, roc_function); - debug_assert_eq!(args.len(), roc_wrapper_function.get_params().len()); + debug_assert_eq!( + arguments_for_call.len(), + roc_wrapper_function.get_params().len() + ); builder.position_at_end(entry); - let call_wrapped = - builder.build_call(roc_wrapper_function, args, "call_wrapped_function"); + let call_wrapped = builder.build_call( + roc_wrapper_function, + arguments_for_call, + "call_wrapped_function", + ); call_wrapped.set_call_convention(FAST_CALL_CONV); call_wrapped.try_as_basic_value().left().unwrap() } else { - let call_unwrapped = builder.build_call(roc_function, args, "call_unwrapped_function"); + let call_unwrapped = + builder.build_call(roc_function, arguments_for_call, "call_unwrapped_function"); call_unwrapped.set_call_convention(FAST_CALL_CONV); let call_unwrapped_result = call_unwrapped.try_as_basic_value().left().unwrap(); - make_good_roc_result(env, call_unwrapped_result) + // make_good_roc_result(env, call_unwrapped_result) + call_unwrapped_result } }; + let output_arg_index = args_length - 1; + let output_arg = c_function .get_nth_param(output_arg_index as u32) .unwrap() .into_pointer_value(); builder.build_store(output_arg, call_result); - builder.build_return(None); + c_function +} + +fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + ident_string: &str, + roc_function: FunctionValue<'ctx>, + arguments: &[Layout<'a>], + return_layout: Layout<'a>, + c_function_name: &str, +) -> FunctionValue<'ctx> { + let context = env.context; + + // a generic version that writes the result into a passed *u8 pointer + if !env.is_gen_test { + expose_function_to_host_help_c_abi_generic( + env, + roc_function, + arguments, + &format!("{}_generic", c_function_name), + ); + } + + let wrapper_return_type = if env.is_gen_test { + context + .struct_type( + &[ + context.i64_type().into(), + roc_function.get_type().get_return_type().unwrap(), + ], + false, + ) + .into() + } else { + roc_function.get_type().get_return_type().unwrap() + }; + + let mut cc_argument_types = Vec::with_capacity_in(arguments.len(), env.arena); + for layout in arguments { + cc_argument_types.push(to_cc_type(env, layout)); + } + + // STEP 1: turn `f : a,b,c -> d` into `f : a,b,c, &d -> {}` if the C abi demands it + let mut argument_types = cc_argument_types; + let return_type = wrapper_return_type; + + let cc_return = to_cc_return(env, &return_layout); + + let c_function_type = match cc_return { + CCReturn::Void if !env.is_gen_test => { + env.context.void_type().fn_type(&argument_types, false) + } + CCReturn::Return if !env.is_gen_test => return_type.fn_type(&argument_types, false), + _ => { + let output_type = return_type.ptr_type(AddressSpace::Generic); + argument_types.push(output_type.into()); + env.context.void_type().fn_type(&argument_types, false) + } + }; + + let c_function = add_func( + env.module, + c_function_name, + c_function_type, + Linkage::External, + C_CALL_CONV, + ); + + let subprogram = env.new_subprogram(c_function_name); + c_function.set_subprogram(subprogram); + + // STEP 2: build the exposed function's body + let builder = env.builder; + let context = env.context; + + let entry = context.append_basic_block(c_function, "entry"); + + builder.position_at_end(entry); + + debug_info_init!(env, c_function); + + // drop the final argument, which is the pointer we write the result into + let args_vector = c_function.get_params(); + let mut args = args_vector.as_slice(); + let args_length = args.len(); + + match cc_return { + CCReturn::Return if !env.is_gen_test => { + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + CCReturn::Void if !env.is_gen_test => { + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + _ => { + args = &args[..args.len() - 1]; + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + } + + let mut arguments_for_call = Vec::with_capacity_in(args.len(), env.arena); + + let it = args.iter().zip(roc_function.get_type().get_param_types()); + for (arg, fastcc_type) in it { + let arg_type = arg.get_type(); + if arg_type == fastcc_type { + // the C and Fast calling conventions agree + arguments_for_call.push(*arg); + } else { + let cast = complex_bitcast_check_size(env, *arg, fastcc_type, "to_fastcc_type"); + arguments_for_call.push(cast); + } + } + + let arguments_for_call = &arguments_for_call.into_bump_slice(); + + let call_result = { + if env.is_gen_test { + let roc_wrapper_function = make_exception_catcher(env, roc_function); + debug_assert_eq!( + arguments_for_call.len(), + roc_wrapper_function.get_params().len() + ); + + builder.position_at_end(entry); + + let call_wrapped = builder.build_call( + roc_wrapper_function, + arguments_for_call, + "call_wrapped_function", + ); + call_wrapped.set_call_convention(FAST_CALL_CONV); + + call_wrapped.try_as_basic_value().left().unwrap() + } else { + let call_unwrapped = + builder.build_call(roc_function, arguments_for_call, "call_unwrapped_function"); + call_unwrapped.set_call_convention(FAST_CALL_CONV); + + let call_unwrapped_result = call_unwrapped.try_as_basic_value().left().unwrap(); + + // make_good_roc_result(env, call_unwrapped_result) + call_unwrapped_result + } + }; + + match cc_return { + CCReturn::Void if !env.is_gen_test => { + // TODO return empty struct here? + builder.build_return(None); + } + CCReturn::Return if !env.is_gen_test => { + builder.build_return(Some(&call_result)); + } + _ => { + let output_arg_index = args_length - 1; + + let output_arg = c_function + .get_nth_param(output_arg_index as u32) + .unwrap() + .into_pointer_value(); + + builder.build_store(output_arg, call_result); + builder.build_return(None); + } + } + // STEP 3: build a {} -> u64 function that gives the size of the return type let size_function_type = env.context.i64_type().fn_type(&[], false); let size_function_name: String = format!("roc__{}_size", ident_string); @@ -3691,7 +3885,14 @@ fn build_proc_header<'a, 'ctx, 'env>( fn_val.set_subprogram(subprogram); if env.exposed_to_host.contains(&symbol) { - expose_function_to_host(env, symbol, fn_val); + let arguments = Vec::from_iter_in(proc.args.iter().map(|(layout, _)| *layout), env.arena); + expose_function_to_host( + env, + symbol, + fn_val, + arguments.into_bump_slice(), + proc.ret_layout, + ); } fn_val @@ -3758,23 +3959,17 @@ pub fn build_closure_caller<'a, 'ctx, 'env>( builder.position_at_end(entry); - let mut parameters = function_value.get_params(); - let output = parameters.pop().unwrap().into_pointer_value(); + let mut evaluator_arguments = function_value.get_params(); - let closure_data = if let Some(closure_data_ptr) = parameters.pop() { - let closure_data = - builder.build_load(closure_data_ptr.into_pointer_value(), "load_closure_data"); + // the final parameter is the output pointer, pop it + let output = evaluator_arguments.pop().unwrap().into_pointer_value(); - env.arena.alloc([closure_data]) as &[_] - } else { - &[] - }; - - let mut parameters = parameters; - - for param in parameters.iter_mut() { - debug_assert!(param.is_pointer_value()); - *param = builder.build_load(param.into_pointer_value(), "load_param"); + // NOTE this may be incorrect in the long run + // here we load any argument that is a pointer + for param in evaluator_arguments.iter_mut() { + if param.is_pointer_value() { + *param = builder.build_load(param.into_pointer_value(), "load_param"); + } } let call_result = if env.is_gen_test { @@ -3783,13 +3978,13 @@ pub fn build_closure_caller<'a, 'ctx, 'env>( function_value, evaluator, evaluator.get_call_conventions(), - closure_data, + &evaluator_arguments, result_type, ) } else { let call = env .builder - .build_call(evaluator, closure_data, "call_function"); + .build_call(evaluator, &evaluator_arguments, "call_function"); call.set_call_convention(evaluator.get_call_conventions()); @@ -4090,7 +4285,7 @@ fn roc_function_call<'a, 'ctx, 'env>( layout_ids: &mut LayoutIds<'a>, transform: FunctionValue<'ctx>, closure_data: BasicValueEnum<'ctx>, - closure_data_layout: Layout<'a>, + lambda_set: LambdaSet<'a>, closure_data_is_owned: bool, argument_layouts: &[Layout<'a>], ) -> RocFunctionCall<'ctx> { @@ -4101,15 +4296,15 @@ fn roc_function_call<'a, 'ctx, 'env>( .build_alloca(closure_data.get_type(), "closure_data_ptr"); env.builder.build_store(closure_data_ptr, closure_data); - let stepper_caller = - build_transform_caller(env, transform, closure_data_layout, argument_layouts) - .as_global_value() - .as_pointer_value(); - - let inc_closure_data = build_inc_n_wrapper(env, layout_ids, &closure_data_layout) + let stepper_caller = build_transform_caller(env, transform, lambda_set, argument_layouts) .as_global_value() .as_pointer_value(); + let inc_closure_data = + build_inc_n_wrapper(env, layout_ids, &lambda_set.runtime_representation()) + .as_global_value() + .as_pointer_value(); + let closure_data_is_owned = env .context .bool_type() @@ -4163,7 +4358,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match list_layout { Layout::Builtin(Builtin::EmptyList) => default, @@ -4175,7 +4370,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4205,7 +4400,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (Layout::Builtin(Builtin::EmptyList), _) => empty_list(env), @@ -4220,7 +4415,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4237,7 +4432,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (list2, list2_layout) = load_symbol_and_layout(scope, &args[1]); let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match (list1_layout, list2_layout, return_layout) { ( @@ -4252,7 +4447,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4281,7 +4476,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (list3, list3_layout) = load_symbol_and_layout(scope, &args[2]); let function = passed_function_at_index!(3); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[4]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[4]); match (list1_layout, list2_layout, list3_layout, return_layout) { ( @@ -4298,7 +4493,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4330,7 +4525,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (Layout::Builtin(Builtin::EmptyList), _) => empty_list(env), @@ -4345,7 +4540,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4363,7 +4558,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match list_layout { Layout::Builtin(Builtin::EmptyList) => empty_list(env), @@ -4375,7 +4570,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4393,7 +4588,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (_, Layout::Builtin(Builtin::EmptyList)) @@ -4409,7 +4604,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4437,7 +4632,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (_, Layout::Builtin(Builtin::EmptyList)) @@ -4453,7 +4648,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4490,7 +4685,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match list_layout { Layout::Builtin(Builtin::EmptyList) => empty_list(env), @@ -4500,7 +4695,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let argument_layouts = &[**element_layout, **element_layout]; let compare_wrapper = - build_compare_wrapper(env, function, *closure_layout, element_layout) + build_compare_wrapper(env, function, closure_layout, element_layout) .as_global_value() .as_pointer_value(); @@ -4509,7 +4704,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4531,7 +4726,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (dict, dict_layout) = load_symbol_and_layout(scope, &args[0]); let (default, default_layout) = load_symbol_and_layout(scope, &args[1]); let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match dict_layout { Layout::Builtin(Builtin::EmptyDict) => { @@ -4546,7 +4741,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4832,7 +5027,7 @@ fn run_low_level<'a, 'ctx, 'env>( Usize | Int128 | Int64 | Int32 | Int16 | Int8 => { build_int_unary_op(env, arg.into_int_value(), arg_builtin, op) } - Float128 | Float64 | Float32 | Float16 => { + Float128 | Float64 | Float32 => { build_float_unary_op(env, arg.into_float_value(), op) } _ => { @@ -4928,7 +5123,7 @@ fn run_low_level<'a, 'ctx, 'env>( "lt_or_gt", ) } - Float128 | Float64 | Float32 | Float16 => { + Float128 | Float64 | Float32 => { let are_equal = env.builder.build_float_compare( FloatPredicate::OEQ, lhs_arg.into_float_value(), @@ -5374,8 +5569,7 @@ fn to_cc_type_builtin<'a, 'ctx, 'env>( | Builtin::Decimal | Builtin::Float128 | Builtin::Float64 - | Builtin::Float32 - | Builtin::Float16 => basic_type_from_builtin(env, builtin), + | Builtin::Float32 => basic_type_from_builtin(env, builtin), Builtin::Str | Builtin::EmptyStr | Builtin::List(_) | Builtin::EmptyList => { env.str_list_c_abi().into() } @@ -5738,7 +5932,7 @@ pub fn build_num_binop<'a, 'ctx, 'env>( rhs_layout, op, ), - Float128 | Float64 | Float32 | Float16 => build_float_binop( + Float128 | Float64 | Float32 => build_float_binop( env, parent, lhs_arg.into_float_value(), diff --git a/compiler/gen_llvm/src/llvm/build_dict.rs b/compiler/gen_llvm/src/llvm/build_dict.rs index 412abf8e3f..75e917a23e 100644 --- a/compiler/gen_llvm/src/llvm/build_dict.rs +++ b/compiler/gen_llvm/src/llvm/build_dict.rs @@ -844,3 +844,17 @@ fn dict_symbol_to_zig_dict<'a, 'ctx, 'env>( ) .into_struct_value() } + +pub fn decref<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + wrapper_struct: StructValue<'ctx>, + alignment: u32, +) { + let pointer = env + .builder + .build_extract_value(wrapper_struct, Builtin::WRAPPER_PTR, "read_list_ptr") + .unwrap() + .into_pointer_value(); + + crate::llvm::refcounting::decref_pointer_check_null(env, pointer, alignment); +} diff --git a/compiler/gen_llvm/src/llvm/build_hash.rs b/compiler/gen_llvm/src/llvm/build_hash.rs index d673251296..d3dc006198 100644 --- a/compiler/gen_llvm/src/llvm/build_hash.rs +++ b/compiler/gen_llvm/src/llvm/build_hash.rs @@ -59,6 +59,15 @@ fn build_hash_layout<'a, 'ctx, 'env>( val.into_struct_value(), ), + Layout::LambdaSet(lambda_set) => build_hash_layout( + env, + layout_ids, + seed, + val, + &lambda_set.runtime_representation(), + when_recursive, + ), + Layout::Union(union_layout) => { build_hash_tag(env, layout_ids, layout, union_layout, seed, val) } @@ -123,7 +132,6 @@ fn hash_builtin<'a, 'ctx, 'env>( | Builtin::Float64 | Builtin::Float32 | Builtin::Float128 - | Builtin::Float16 | Builtin::Decimal | Builtin::Usize => { let hash_bytes = store_and_use_as_u8_ptr(env, val, layout); diff --git a/compiler/gen_llvm/src/llvm/build_list.rs b/compiler/gen_llvm/src/llvm/build_list.rs index 8cbdd71083..2fe94d4cb5 100644 --- a/compiler/gen_llvm/src/llvm/build_list.rs +++ b/compiler/gen_llvm/src/llvm/build_list.rs @@ -1136,3 +1136,17 @@ pub fn store_list<'a, 'ctx, 'env>( "cast_collection", ) } + +pub fn decref<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + wrapper_struct: StructValue<'ctx>, + alignment: u32, +) { + let (_, pointer) = load_list( + env.builder, + wrapper_struct, + env.context.i8_type().ptr_type(AddressSpace::Generic), + ); + + crate::llvm::refcounting::decref_pointer_check_null(env, pointer, alignment); +} diff --git a/compiler/gen_llvm/src/llvm/build_str.rs b/compiler/gen_llvm/src/llvm/build_str.rs index 87a8c50c57..80bfa0fa3a 100644 --- a/compiler/gen_llvm/src/llvm/build_str.rs +++ b/compiler/gen_llvm/src/llvm/build_str.rs @@ -1,5 +1,5 @@ use crate::llvm::bitcode::{call_bitcode_fn, call_void_bitcode_fn}; -use crate::llvm::build::{complex_bitcast, struct_from_fields, Env, Scope}; +use crate::llvm::build::{complex_bitcast, Env, Scope}; use crate::llvm::build_list::{allocate_list, call_bitcode_fn_returns_list, store_list}; use inkwell::builder::Builder; use inkwell::values::{BasicValueEnum, FunctionValue, IntValue, PointerValue, StructValue}; @@ -268,48 +268,19 @@ fn decode_from_utf8_result<'a, 'ctx, 'env>( let ctx = env.context; let fields = match env.ptr_bytes { - 8 => [ + 8 | 4 => [ env.ptr_int().into(), super::convert::zig_str_type(env).into(), env.context.bool_type().into(), ctx.i8_type().into(), ], - 4 => [ - super::convert::zig_str_type(env).into(), - env.ptr_int().into(), - env.context.bool_type().into(), - ctx.i8_type().into(), - ], _ => unreachable!(), }; let record_type = env.context.struct_type(&fields, false); match env.ptr_bytes { - 8 => { - let zig_struct = builder - .build_load(pointer, "load_utf8_validate_bytes_result") - .into_struct_value(); - - let string = builder - .build_extract_value(zig_struct, 0, "string") - .unwrap(); - - let byte_index = builder - .build_extract_value(zig_struct, 1, "byte_index") - .unwrap(); - - let is_ok = builder.build_extract_value(zig_struct, 2, "is_ok").unwrap(); - - let problem_code = builder - .build_extract_value(zig_struct, 3, "problem_code") - .unwrap(); - - let values = [byte_index, string, is_ok, problem_code]; - - struct_from_fields(env, record_type, values.iter().copied().enumerate()) - } - 4 => { + 8 | 4 => { let result_ptr_cast = env .builder .build_bitcast( diff --git a/compiler/gen_llvm/src/llvm/compare.rs b/compiler/gen_llvm/src/llvm/compare.rs index e2fa27604c..1ca6178c0b 100644 --- a/compiler/gen_llvm/src/llvm/compare.rs +++ b/compiler/gen_llvm/src/llvm/compare.rs @@ -103,7 +103,6 @@ fn build_eq_builtin<'a, 'ctx, 'env>( Builtin::Float128 => float_cmp(FloatPredicate::OEQ, "eq_f128"), Builtin::Float64 => float_cmp(FloatPredicate::OEQ, "eq_f64"), Builtin::Float32 => float_cmp(FloatPredicate::OEQ, "eq_f32"), - Builtin::Float16 => float_cmp(FloatPredicate::OEQ, "eq_f16"), Builtin::Str => str_equal(env, lhs_val, rhs_val), Builtin::List(elem) => build_list_eq( @@ -156,6 +155,8 @@ fn build_eq<'a, 'ctx, 'env>( rhs_val.into_struct_value(), ), + Layout::LambdaSet(_) => unreachable!("cannot compare closures"), + Layout::Union(union_layout) => build_tag_eq( env, layout_ids, @@ -245,7 +246,6 @@ fn build_neq_builtin<'a, 'ctx, 'env>( Builtin::Float128 => float_cmp(FloatPredicate::ONE, "neq_f128"), Builtin::Float64 => float_cmp(FloatPredicate::ONE, "neq_f64"), Builtin::Float32 => float_cmp(FloatPredicate::ONE, "neq_f32"), - Builtin::Float16 => float_cmp(FloatPredicate::ONE, "neq_f16"), Builtin::Str => { let is_equal = str_equal(env, lhs_val, rhs_val).into_int_value(); @@ -336,6 +336,7 @@ fn build_neq<'a, 'ctx, 'env>( Layout::RecursivePointer => { unreachable!("recursion pointers should never be compared directly") } + Layout::LambdaSet(_) => unreachable!("cannot compare closure"), } } diff --git a/compiler/gen_llvm/src/llvm/convert.rs b/compiler/gen_llvm/src/llvm/convert.rs index 7192400542..d32216c10a 100644 --- a/compiler/gen_llvm/src/llvm/convert.rs +++ b/compiler/gen_llvm/src/llvm/convert.rs @@ -27,6 +27,7 @@ pub fn basic_type_from_layout<'a, 'ctx, 'env>( match layout { Struct(sorted_fields) => basic_type_from_record(env, sorted_fields), + LambdaSet(lambda_set) => basic_type_from_layout(env, &lambda_set.runtime_representation()), Union(union_layout) => { use UnionLayout::*; @@ -96,7 +97,6 @@ pub fn basic_type_from_builtin<'a, 'ctx, 'env>( Float128 => context.f128_type().as_basic_type_enum(), Float64 => context.f64_type().as_basic_type_enum(), Float32 => context.f32_type().as_basic_type_enum(), - Float16 => context.f16_type().as_basic_type_enum(), Dict(_, _) | EmptyDict => zig_dict_type(env).into(), Set(_) | EmptySet => zig_dict_type(env).into(), List(_) | EmptyList => zig_list_type(env).into(), diff --git a/compiler/gen_llvm/src/llvm/refcounting.rs b/compiler/gen_llvm/src/llvm/refcounting.rs index 4e96a106f7..8a89c924d7 100644 --- a/compiler/gen_llvm/src/llvm/refcounting.rs +++ b/compiler/gen_llvm/src/llvm/refcounting.rs @@ -84,7 +84,7 @@ impl<'ctx> PointerToRefcount<'ctx> { } } - pub fn from_list_wrapper(env: &Env<'_, 'ctx, '_>, list_wrapper: StructValue<'ctx>) -> Self { + fn from_list_wrapper(env: &Env<'_, 'ctx, '_>, list_wrapper: StructValue<'ctx>) -> Self { let data_ptr = env .builder .build_extract_value(list_wrapper, Builtin::WRAPPER_PTR, "read_list_ptr") @@ -102,7 +102,7 @@ impl<'ctx> PointerToRefcount<'ctx> { .build_int_compare(IntPredicate::EQ, current, one, "is_one") } - pub fn get_refcount<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>) -> IntValue<'ctx> { + fn get_refcount<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>) -> IntValue<'ctx> { env.builder .build_load(self.value, "get_refcount") .into_int_value() @@ -220,25 +220,57 @@ impl<'ctx> PointerToRefcount<'ctx> { debug_info_init!(env, parent); - let alignment = env.context.i32_type().const_int(alignment as _, false); - - call_void_bitcode_fn( + decref_pointer( env, - &[ - env.builder.build_bitcast( - parent.get_nth_param(0).unwrap(), - env.ptr_int().ptr_type(AddressSpace::Generic), - "foo", - ), - alignment.into(), - ], - roc_builtins::bitcode::UTILS_DECREF, + parent.get_nth_param(0).unwrap().into_pointer_value(), + alignment, ); builder.build_return(None); } } +fn decref_pointer<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + pointer: PointerValue<'ctx>, + alignment: u32, +) { + let alignment = env.context.i32_type().const_int(alignment as _, false); + call_void_bitcode_fn( + env, + &[ + env.builder.build_bitcast( + pointer, + env.ptr_int().ptr_type(AddressSpace::Generic), + "to_isize_ptr", + ), + alignment.into(), + ], + roc_builtins::bitcode::UTILS_DECREF, + ); +} + +/// Assumes a pointer to the refcount +pub fn decref_pointer_check_null<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + pointer: PointerValue<'ctx>, + alignment: u32, +) { + let alignment = env.context.i32_type().const_int(alignment as _, false); + call_void_bitcode_fn( + env, + &[ + env.builder.build_bitcast( + pointer, + env.context.i8_type().ptr_type(AddressSpace::Generic), + "to_i8_ptr", + ), + alignment.into(), + ], + roc_builtins::bitcode::UTILS_DECREF_CHECK_NULL, + ); +} + fn modify_refcount_struct<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, layout_ids: &mut LayoutIds<'a>, @@ -594,6 +626,14 @@ fn modify_refcount_layout_build_function<'a, 'ctx, 'env>( Some(function) } }, + LambdaSet(lambda_set) => modify_refcount_layout_build_function( + env, + parent, + layout_ids, + mode, + when_recursive, + &lambda_set.runtime_representation(), + ), } } @@ -1093,7 +1133,6 @@ fn build_rec_union_help<'a, 'ctx, 'env>( fn_val: FunctionValue<'ctx>, ) { let tags = union_layout_tags(env.arena, &union_layout); - let is_nullable = union_layout.is_nullable(); debug_assert!(!tags.is_empty()); let context = &env.context; @@ -1126,7 +1165,7 @@ fn build_rec_union_help<'a, 'ctx, 'env>( let should_recurse_block = env.context.append_basic_block(parent, "should_recurse"); let ctx = env.context; - if is_nullable { + if union_layout.is_nullable() { let is_null = env.builder.build_is_null(value_ptr, "is_null"); let then_block = ctx.append_basic_block(parent, "then"); @@ -1201,6 +1240,12 @@ enum DecOrReuse { Reuse, } +fn fields_need_no_refcounting(field_layouts: &[Layout]) -> bool { + !field_layouts + .iter() + .any(|x| x.is_refcounted() || x.contains_refcounted()) +} + #[allow(clippy::too_many_arguments)] fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, @@ -1220,21 +1265,6 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( let call_mode = mode_to_call_mode(decrement_fn, mode); let builder = env.builder; - // branches that are not/don't contain anything refcounted - // if there is only one branch, we don't need to switch - let switch_needed: bool = (|| { - for field_layouts in tags.iter() { - // if none of the fields are or contain anything refcounted, just move on - if !field_layouts - .iter() - .any(|x| x.is_refcounted() || x.contains_refcounted()) - { - return true; - } - } - false - })(); - // next, make a jump table for all possible values of the tag_id let mut cases = Vec::with_capacity_in(tags.len(), env.arena); @@ -1243,10 +1273,7 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( for (tag_id, field_layouts) in tags.iter().enumerate() { // if none of the fields are or contain anything refcounted, just move on - if !field_layouts - .iter() - .any(|x| x.is_refcounted() || x.contains_refcounted()) - { + if fields_need_no_refcounting(field_layouts) { continue; } @@ -1346,11 +1373,13 @@ fn build_rec_union_recursive_decrement<'a, 'ctx, 'env>( cases.reverse(); - if cases.len() == 1 && !switch_needed { - // there is only one tag in total; we don't need a switch - // this is essential for nullable unwrapped layouts, - // because the `else` branch below would try to read its - // (nonexistant) tag id + if matches!( + union_layout, + UnionLayout::NullableUnwrapped { .. } | UnionLayout::NonNullableUnwrapped { .. } + ) { + debug_assert_eq!(cases.len(), 1); + + // in this case, don't switch, because the `else` branch below would try to read the (nonexistant) tag id let (_, only_branch) = cases.pop().unwrap(); env.builder.build_unconditional_branch(only_branch); } else { @@ -1444,7 +1473,6 @@ fn build_reuse_rec_union_help<'a, 'ctx, 'env>( dec_function: FunctionValue<'ctx>, ) { let tags = union_layout_tags(env.arena, &union_layout); - let is_nullable = union_layout.is_nullable(); debug_assert!(!tags.is_empty()); @@ -1478,7 +1506,7 @@ fn build_reuse_rec_union_help<'a, 'ctx, 'env>( let should_recurse_block = env.context.append_basic_block(parent, "should_recurse"); let ctx = env.context; - if is_nullable { + if union_layout.is_nullable() { let is_null = env.builder.build_is_null(value_ptr, "is_null"); let then_block = ctx.append_basic_block(parent, "then"); @@ -1700,68 +1728,3 @@ fn modify_refcount_union_help<'a, 'ctx, 'env>( // this function returns void builder.build_return(None); } - -pub fn refcount_is_one_comparison<'a, 'ctx, 'env>( - env: &Env<'a, 'ctx, 'env>, - refcount: IntValue<'ctx>, -) -> IntValue<'ctx> { - env.builder.build_int_compare( - IntPredicate::EQ, - refcount, - refcount_1(env.context, env.ptr_bytes), - "refcount_one_check", - ) -} - -pub fn list_get_refcount_ptr<'a, 'ctx, 'env>( - env: &Env<'a, 'ctx, 'env>, - layout: &Layout<'a>, - list_wrapper: StructValue<'ctx>, -) -> PointerValue<'ctx> { - // fetch the pointer to the array data, as an integer - let ptr_as_int = env - .builder - .build_extract_value(list_wrapper, Builtin::WRAPPER_PTR, "read_list_ptr") - .unwrap() - .into_int_value(); - - get_refcount_ptr_help(env, layout, ptr_as_int) -} - -pub fn refcount_offset<'a, 'ctx, 'env>(env: &Env<'a, 'ctx, 'env>, layout: &Layout<'a>) -> u64 { - let value_bytes = layout.stack_size(env.ptr_bytes) as u64; - - match layout { - Layout::Builtin(Builtin::List(_)) => env.ptr_bytes as u64, - Layout::Builtin(Builtin::Str) => env.ptr_bytes as u64, - Layout::RecursivePointer | Layout::Union(_) => env.ptr_bytes as u64, - _ => (env.ptr_bytes as u64).max(value_bytes), - } -} - -fn get_refcount_ptr_help<'a, 'ctx, 'env>( - env: &Env<'a, 'ctx, 'env>, - layout: &Layout<'a>, - ptr_as_int: IntValue<'ctx>, -) -> PointerValue<'ctx> { - let builder = env.builder; - let ctx = env.context; - - let offset = refcount_offset(env, layout); - - // pointer to usize - let refcount_type = ptr_int(ctx, env.ptr_bytes); - - // subtract offset, to access the refcount - let refcount_ptr = builder.build_int_sub( - ptr_as_int, - refcount_type.const_int(offset, false), - "make_refcount_ptr", - ); - - builder.build_int_to_ptr( - refcount_ptr, - refcount_type.ptr_type(AddressSpace::Generic), - "get_refcount_ptr", - ) -} diff --git a/compiler/gen_wasm/README.md b/compiler/gen_wasm/README.md index 3e453b21e2..50018f8758 100644 --- a/compiler/gen_wasm/README.md +++ b/compiler/gen_wasm/README.md @@ -5,6 +5,12 @@ - Initial bringup - Get a wasm backend working for some of the number tests. - Use a separate `gen_wasm` directory for now, to avoid trying to do bringup and integration at the same time. +- Improve the fundamentals + - [x] Come up with a way to do control flow + - [x] Flesh out the details of value representations between local variables and stack memory + - [ ] Set up a way to write tests with any return value rather than just i64 and f64 + - [ ] Figure out relocations for linking object files + - [ ] Think about the Wasm module builder library we're using, are we happy with it? - Integration - Move wasm files to `gen_dev/src/wasm` - Share tests between wasm and x64, with some way of saying which tests work on which backends, and dispatching to different eval helpers based on that. diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index b66c4847d7..0b03f4aea8 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -1,12 +1,16 @@ use parity_wasm::builder; use parity_wasm::builder::{CodeLocation, ModuleBuilder}; -use parity_wasm::elements::{Instruction, Instruction::*, Instructions, Local, ValueType}; +use parity_wasm::elements::{ + BlockType, Instruction, Instruction::*, Instructions, Local, ValueType, +}; use roc_collections::all::MutMap; use roc_module::low_level::LowLevel; use roc_module::symbol::Symbol; -use roc_mono::ir::{CallType, Expr, Literal, Proc, Stmt}; -use roc_mono::layout::{Builtin, Layout}; +use roc_mono::ir::{CallType, Expr, JoinPointId, Literal, Proc, Stmt}; +use roc_mono::layout::{Builtin, Layout, UnionLayout}; + +use crate::*; // Don't allocate any constant data at address zero or near it. Would be valid, but bug-prone. // Follow Emscripten's example by using 1kB (4 bytes would probably do) @@ -21,24 +25,126 @@ struct LabelId(u32); #[derive(Debug)] struct SymbolStorage(LocalId, WasmLayout); +// See README for background information on Wasm locals, memory and function calls #[derive(Debug)] -struct WasmLayout { - value_type: ValueType, - stack_memory: u32, +pub enum WasmLayout { + // Most number types can fit in a Wasm local without any stack memory. + // Roc i8 is represented as an i32 local. Store the type and the original size. + LocalOnly(ValueType, u32), + + // A `local` pointing to stack memory + StackMemory(u32), + + // A `local` pointing to heap memory + HeapMemory, } impl WasmLayout { - fn new(layout: &Layout) -> Result { + fn new(layout: &Layout) -> Self { + use ValueType::*; + let size = layout.stack_size(PTR_SIZE); match layout { - Layout::Builtin(Builtin::Int64) => Ok(Self { - value_type: ValueType::I64, - stack_memory: 0, - }), - Layout::Builtin(Builtin::Float64) => Ok(Self { - value_type: ValueType::F64, - stack_memory: 0, - }), - x => Err(format!("layout, {:?}, not implemented yet", x)), + Layout::Builtin(Builtin::Int128) => Self::StackMemory(size), + Layout::Builtin(Builtin::Int64) => Self::LocalOnly(I64, size), + Layout::Builtin(Builtin::Int32) => Self::LocalOnly(I32, size), + Layout::Builtin(Builtin::Int16) => Self::LocalOnly(I32, size), + Layout::Builtin(Builtin::Int8) => Self::LocalOnly(I32, size), + Layout::Builtin(Builtin::Int1) => Self::LocalOnly(I32, size), + Layout::Builtin(Builtin::Usize) => Self::LocalOnly(I32, size), + Layout::Builtin(Builtin::Decimal) => Self::StackMemory(size), + Layout::Builtin(Builtin::Float128) => Self::StackMemory(size), + Layout::Builtin(Builtin::Float64) => Self::LocalOnly(F64, size), + Layout::Builtin(Builtin::Float32) => Self::LocalOnly(F32, size), + Layout::Builtin(Builtin::Str) => Self::StackMemory(size), + Layout::Builtin(Builtin::Dict(_, _)) => Self::StackMemory(size), + Layout::Builtin(Builtin::Set(_)) => Self::StackMemory(size), + Layout::Builtin(Builtin::List(_)) => Self::StackMemory(size), + Layout::Builtin(Builtin::EmptyStr) => Self::StackMemory(size), + Layout::Builtin(Builtin::EmptyList) => Self::StackMemory(size), + Layout::Builtin(Builtin::EmptyDict) => Self::StackMemory(size), + Layout::Builtin(Builtin::EmptySet) => Self::StackMemory(size), + Layout::LambdaSet(lambda_set) => WasmLayout::new(&lambda_set.runtime_representation()), + Layout::Struct(_) => Self::StackMemory(size), + Layout::Union(UnionLayout::NonRecursive(_)) => Self::StackMemory(size), + Layout::Union(UnionLayout::Recursive(_)) => Self::HeapMemory, + Layout::Union(UnionLayout::NonNullableUnwrapped(_)) => Self::HeapMemory, + Layout::Union(UnionLayout::NullableWrapped { .. }) => Self::HeapMemory, + Layout::Union(UnionLayout::NullableUnwrapped { .. }) => Self::HeapMemory, + Layout::RecursivePointer => Self::HeapMemory, + } + } + + fn value_type(&self) -> ValueType { + match self { + Self::LocalOnly(type_, _) => *type_, + _ => PTR_TYPE, + } + } + + fn stack_memory(&self) -> u32 { + match self { + Self::StackMemory(size) => *size, + _ => 0, + } + } + + #[allow(dead_code)] + fn load(&self, offset: u32) -> Result { + use crate::backend::WasmLayout::*; + use ValueType::*; + + match self { + LocalOnly(I32, 4) => Ok(I32Load(ALIGN_4, offset)), + LocalOnly(I32, 2) => Ok(I32Load16S(ALIGN_2, offset)), + LocalOnly(I32, 1) => Ok(I32Load8S(ALIGN_1, offset)), + LocalOnly(I64, 8) => Ok(I64Load(ALIGN_8, offset)), + LocalOnly(F64, 8) => Ok(F64Load(ALIGN_8, offset)), + LocalOnly(F32, 4) => Ok(F32Load(ALIGN_4, offset)), + + // LocalOnly(F32, 2) => Ok(), // convert F16 to F32 (lowlevel function? Wasm-only?) + // StackMemory(size) => Ok(), // would this be some kind of memcpy in the IR? + HeapMemory => { + if PTR_TYPE == I64 { + Ok(I64Load(ALIGN_8, offset)) + } else { + Ok(I32Load(ALIGN_4, offset)) + } + } + + _ => Err(format!( + "Failed to generate load instruction for WasmLayout {:?}", + self + )), + } + } + + #[allow(dead_code)] + fn store(&self, offset: u32) -> Result { + use crate::backend::WasmLayout::*; + use ValueType::*; + + match self { + LocalOnly(I32, 4) => Ok(I32Store(ALIGN_4, offset)), + LocalOnly(I32, 2) => Ok(I32Store16(ALIGN_2, offset)), + LocalOnly(I32, 1) => Ok(I32Store8(ALIGN_1, offset)), + LocalOnly(I64, 8) => Ok(I64Store(ALIGN_8, offset)), + LocalOnly(F64, 8) => Ok(F64Store(ALIGN_8, offset)), + LocalOnly(F32, 4) => Ok(F32Store(ALIGN_4, offset)), + + // LocalOnly(F32, 2) => Ok(), // convert F32 to F16 (lowlevel function? Wasm-only?) + // StackMemory(size) => Ok(), // would this be some kind of memcpy in the IR? + HeapMemory => { + if PTR_TYPE == I64 { + Ok(I64Store(ALIGN_8, offset)) + } else { + Ok(I32Store(ALIGN_4, offset)) + } + } + + _ => Err(format!( + "Failed to generate store instruction for WasmLayout {:?}", + self + )), } } } @@ -61,7 +167,9 @@ pub struct WasmBackend<'a> { // Functions: internal state & IR mappings stack_memory: u32, symbol_storage_map: MutMap, - // joinpoint_label_map: MutMap, + /// how many blocks deep are we (used for jumps) + block_depth: u32, + joinpoint_label_map: MutMap)>, } impl<'a> WasmBackend<'a> { @@ -84,7 +192,8 @@ impl<'a> WasmBackend<'a> { // Functions: internal state & IR mappings stack_memory: 0, symbol_storage_map: MutMap::default(), - // joinpoint_label_map: MutMap::default(), + block_depth: 0, + joinpoint_label_map: MutMap::default(), } } @@ -101,21 +210,21 @@ impl<'a> WasmBackend<'a> { } pub fn build_proc(&mut self, proc: Proc<'a>, sym: Symbol) -> Result { - let ret_layout = WasmLayout::new(&proc.ret_layout)?; - if ret_layout.stack_memory > 0 { - // TODO: if returning a struct by value, add an extra argument for a pointer to callee's stack memory + let ret_layout = WasmLayout::new(&proc.ret_layout); + + if let WasmLayout::StackMemory { .. } = ret_layout { return Err(format!( - "Not yet implemented: Return in stack memory for non-primtitive layouts like {:?}", - proc.ret_layout + "Not yet implemented: Returning values to callee stack memory {:?} {:?}", + proc.name, sym )); } - self.ret_type = ret_layout.value_type; + self.ret_type = ret_layout.value_type(); self.arg_types.reserve(proc.args.len()); for (layout, symbol) in proc.args { - let wasm_layout = WasmLayout::new(layout)?; - self.arg_types.push(wasm_layout.value_type); + let wasm_layout = WasmLayout::new(layout); + self.arg_types.push(wasm_layout.value_type()); self.insert_local(wasm_layout, *symbol); } @@ -147,10 +256,10 @@ impl<'a> WasmBackend<'a> { } fn insert_local(&mut self, layout: WasmLayout, symbol: Symbol) -> LocalId { - self.stack_memory += layout.stack_memory; + self.stack_memory += layout.stack_memory(); let index = self.symbol_storage_map.len(); if index >= self.arg_types.len() { - self.locals.push(Local::new(1, layout.value_type)); + self.locals.push(Local::new(1, layout.value_type())); } let local_id = LocalId(index as u32); let storage = SymbolStorage(local_id, layout); @@ -174,6 +283,27 @@ impl<'a> WasmBackend<'a> { Ok(()) } + /// start a loop that leaves a value on the stack + fn start_loop_with_return(&mut self, value_type: ValueType) { + self.block_depth += 1; + + // self.instructions.push(Loop(BlockType::NoResult)); + self.instructions.push(Loop(BlockType::Value(value_type))); + } + + fn start_block(&mut self) { + self.block_depth += 1; + + // Our blocks always end with a `return` or `br`, + // so they never leave extra values on the stack + self.instructions.push(Block(BlockType::NoResult)); + } + + fn end_block(&mut self) { + self.block_depth -= 1; + self.instructions.push(End); + } + fn build_stmt(&mut self, stmt: &Stmt<'a>, ret_layout: &Layout<'a>) -> Result<(), String> { match stmt { // This pattern is a simple optimisation to get rid of one local and two instructions per proc. @@ -185,7 +315,7 @@ impl<'a> WasmBackend<'a> { } Stmt::Let(sym, expr, layout, following) => { - let wasm_layout = WasmLayout::new(layout)?; + let wasm_layout = WasmLayout::new(layout); let local_id = self.insert_local(wasm_layout, *sym); self.build_expr(sym, expr, layout)?; @@ -207,6 +337,113 @@ impl<'a> WasmBackend<'a> { )) } } + + Stmt::Switch { + cond_symbol, + cond_layout: _, + branches, + default_branch, + ret_layout: _, + } => { + // NOTE currently implemented as a series of conditional jumps + // We may be able to improve this in the future with `Select` + // or `BrTable` + + // create (number_of_branches - 1) new blocks. + for _ in 0..branches.len() { + self.start_block() + } + + // the LocalId of the symbol that we match on + let matched_on = match self.symbol_storage_map.get(cond_symbol) { + Some(SymbolStorage(local_id, _)) => local_id.0, + None => unreachable!("symbol not defined: {:?}", cond_symbol), + }; + + // then, we jump whenever the value under scrutiny is equal to the value of a branch + for (i, (value, _, _)) in branches.iter().enumerate() { + // put the cond_symbol on the top of the stack + self.instructions.push(GetLocal(matched_on)); + + self.instructions.push(I32Const(*value as i32)); + + // compare the 2 topmost values + self.instructions.push(I32Eq); + + // "break" out of `i` surrounding blocks + self.instructions.push(BrIf(i as u32)); + } + + // if we never jumped because a value matched, we're in the default case + self.build_stmt(default_branch.1, ret_layout)?; + + // now put in the actual body of each branch in order + // (the first branch would have broken out of 1 block, + // hence we must generate its code first) + for (_, _, branch) in branches.iter() { + self.end_block(); + + self.build_stmt(branch, ret_layout)?; + } + + Ok(()) + } + Stmt::Join { + id, + parameters, + body, + remainder, + } => { + // make locals for join pointer parameters + let mut jp_parameter_local_ids = std::vec::Vec::with_capacity(parameters.len()); + for parameter in parameters.iter() { + let wasm_layout = WasmLayout::new(¶meter.layout); + let local_id = self.insert_local(wasm_layout, parameter.symbol); + + jp_parameter_local_ids.push(local_id); + } + + self.start_block(); + + self.joinpoint_label_map + .insert(*id, (self.block_depth, jp_parameter_local_ids)); + + self.build_stmt(remainder, ret_layout)?; + + self.end_block(); + + // A `return` inside of a `loop` seems to make it so that the `loop` itself + // also "returns" (so, leaves on the stack) a value of the return type. + let return_wasm_layout = WasmLayout::new(ret_layout); + self.start_loop_with_return(return_wasm_layout.value_type()); + + self.build_stmt(body, ret_layout)?; + + // ends the loop + self.end_block(); + + Ok(()) + } + Stmt::Jump(id, arguments) => { + let (target, locals) = &self.joinpoint_label_map[id]; + + // put the arguments on the stack + for (symbol, local_id) in arguments.iter().zip(locals.iter()) { + let argument = match self.symbol_storage_map.get(symbol) { + Some(SymbolStorage(local_id, _)) => local_id.0, + None => unreachable!("symbol not defined: {:?}", symbol), + }; + + self.instructions.push(GetLocal(argument)); + self.instructions.push(SetLocal(local_id.0)); + } + + // jump + let levels = self.block_depth - target; + self.instructions.push(Br(levels)); + + Ok(()) + } x => Err(format!("statement not yet implemented: {:?}", x)), } } @@ -218,7 +455,7 @@ impl<'a> WasmBackend<'a> { layout: &Layout<'a>, ) -> Result<(), String> { match expr { - Expr::Literal(lit) => self.load_literal(lit), + Expr::Literal(lit) => self.load_literal(lit, layout), Expr::Call(roc_mono::ir::Call { call_type, @@ -246,15 +483,38 @@ impl<'a> WasmBackend<'a> { } } - fn load_literal(&mut self, lit: &Literal<'a>) -> Result<(), String> { + fn load_literal(&mut self, lit: &Literal<'a>, layout: &Layout<'a>) -> Result<(), String> { match lit { + Literal::Bool(x) => { + self.instructions.push(I32Const(*x as i32)); + Ok(()) + } + Literal::Byte(x) => { + self.instructions.push(I32Const(*x as i32)); + Ok(()) + } Literal::Int(x) => { - self.instructions.push(I64Const(*x as i64)); + let instruction = match layout { + Layout::Builtin(Builtin::Int64) => I64Const(*x as i64), + Layout::Builtin( + Builtin::Int32 + | Builtin::Int16 + | Builtin::Int8 + | Builtin::Int1 + | Builtin::Usize, + ) => I32Const(*x as i32), + x => panic!("loading literal, {:?}, is not yet implemented", x), + }; + self.instructions.push(instruction); Ok(()) } Literal::Float(x) => { - let val: f64 = *x; - self.instructions.push(F64Const(val.to_bits())); + let instruction = match layout { + Layout::Builtin(Builtin::Float64) => F64Const((*x as f64).to_bits()), + Layout::Builtin(Builtin::Float32) => F32Const((*x as f32).to_bits()), + x => panic!("loading literal, {:?}, is not yet implemented", x), + }; + self.instructions.push(instruction); Ok(()) } x => Err(format!("loading literal, {:?}, is not yet implemented", x)), @@ -270,8 +530,8 @@ impl<'a> WasmBackend<'a> { for arg in args { self.load_from_symbol(arg)?; } - let wasm_layout = WasmLayout::new(return_layout)?; - self.build_instructions_lowlevel(lowlevel, wasm_layout.value_type)?; + let wasm_layout = WasmLayout::new(return_layout); + self.build_instructions_lowlevel(lowlevel, wasm_layout.value_type())?; Ok(()) } @@ -293,6 +553,22 @@ impl<'a> WasmBackend<'a> { ValueType::F32 => &[F32Add], ValueType::F64 => &[F64Add], }, + LowLevel::NumSub => match return_value_type { + ValueType::I32 => &[I32Sub], + ValueType::I64 => &[I64Sub], + ValueType::F32 => &[F32Sub], + ValueType::F64 => &[F64Sub], + }, + LowLevel::NumMul => match return_value_type { + ValueType::I32 => &[I32Mul], + ValueType::I64 => &[I64Mul], + ValueType::F32 => &[F32Mul], + ValueType::F64 => &[F64Mul], + }, + LowLevel::NumGt => { + // needs layout of the argument to be implemented fully + &[I32GtS] + } _ => { return Err(format!("unsupported low-level op {:?}", lowlevel)); } diff --git a/compiler/gen_wasm/src/lib.rs b/compiler/gen_wasm/src/lib.rs index 77836ec651..37eb3c1d5b 100644 --- a/compiler/gen_wasm/src/lib.rs +++ b/compiler/gen_wasm/src/lib.rs @@ -3,7 +3,7 @@ pub mod from_wasm32_memory; use bumpalo::Bump; use parity_wasm::builder; -use parity_wasm::elements::Internal; +use parity_wasm::elements::{Instruction, Internal, ValueType}; use roc_collections::all::{MutMap, MutSet}; use roc_module::symbol::{Interns, Symbol}; @@ -12,6 +12,17 @@ use roc_mono::layout::LayoutIds; use crate::backend::WasmBackend; +const PTR_SIZE: u32 = 4; +const PTR_TYPE: ValueType = ValueType::I32; + +// All usages of these alignment constants take u32, so an enum wouldn't add any safety. +pub const ALIGN_1: u32 = 0; +pub const ALIGN_2: u32 = 1; +pub const ALIGN_4: u32 = 2; +pub const ALIGN_8: u32 = 3; + +pub const STACK_POINTER_GLOBAL_ID: u32 = 0; + pub struct Env<'a> { pub arena: &'a Bump, // not really using this much, parity_wasm works with std::vec a lot pub interns: Interns, @@ -21,12 +32,37 @@ pub struct Env<'a> { pub fn build_module<'a>( env: &'a Env, procedures: MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>, -) -> Result, String> { +) -> Result, String> { + let (builder, _) = build_module_help(env, procedures)?; + let module = builder.build(); + module + .to_bytes() + .map_err(|e| -> String { format!("Error serialising Wasm module {:?}", e) }) +} + +pub fn build_module_help<'a>( + env: &'a Env, + procedures: MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>, +) -> Result<(builder::ModuleBuilder, u32), String> { let mut backend = WasmBackend::new(); let mut layout_ids = LayoutIds::default(); + // Sort procedures by occurrence order + // + // We sort by the "name", but those are interned strings, and the name that is + // interned first will have a lower number. + // + // But, the name that occurs first is always `main` because it is in the (implicit) + // file header. Therefore sorting high to low will put other functions before main + // + // This means that for now other functions in the file have to be ordered "in reverse": if A + // uses B, then the name of A must first occur after the first occurrence of the name of B + let mut procedures: std::vec::Vec<_> = procedures.into_iter().collect(); + procedures.sort_by(|a, b| b.0 .0.cmp(&a.0 .0)); + + let mut function_index: u32 = 0; for ((sym, layout), proc) in procedures { - let function_index = backend.build_proc(proc, sym)?; + function_index = backend.build_proc(proc, sym)?; if env.exposed_to_host.contains(&sym) { let fn_name = layout_ids .get_toplevel(sym, &layout) @@ -41,6 +77,11 @@ pub fn build_module<'a>( } } + // Because of the sorting above, we know the last function in the `for` is the main function. + // Here we grab its index and return it, so that the test_wrapper is able to call it. + // This is a workaround until we implement object files with symbols and relocations. + let main_function_index = function_index; + const MIN_MEMORY_SIZE_KB: u32 = 1024; const PAGE_SIZE_KB: u32 = 64; @@ -48,15 +89,18 @@ pub fn build_module<'a>( .with_min(MIN_MEMORY_SIZE_KB / PAGE_SIZE_KB) .build(); backend.builder.push_memory(memory); - let memory_export = builder::export() .field("memory") .with_internal(Internal::Memory(0)) .build(); backend.builder.push_export(memory_export); - let module = backend.builder.build(); - module - .to_bytes() - .map_err(|e| -> String { format!("Error serialising Wasm module {:?}", e) }) + let stack_pointer_global = builder::global() + .with_type(PTR_TYPE) + .mutable() + .init_expr(Instruction::I32Const((MIN_MEMORY_SIZE_KB * 1024) as i32)) + .build(); + backend.builder.push_global(stack_pointer_global); + + Ok((backend.builder, main_function_index)) } diff --git a/compiler/gen_wasm/tests/helpers/eval.rs b/compiler/gen_wasm/tests/helpers/eval.rs index d380c77e3a..f29a360094 100644 --- a/compiler/gen_wasm/tests/helpers/eval.rs +++ b/compiler/gen_wasm/tests/helpers/eval.rs @@ -1,6 +1,10 @@ use roc_can::builtins::builtin_defs_map; use roc_collections::all::{MutMap, MutSet}; // use roc_std::{RocDec, RocList, RocOrder, RocStr}; +use crate::helpers::wasm32_test_result::Wasm32TestResult; +use roc_gen_wasm::from_wasm32_memory::FromWasm32Memory; + +const TEST_WRAPPER_NAME: &str = "test_wrapper"; fn promote_expr_to_module(src: &str) -> String { let mut buffer = String::from("app \"test\" provides [ main ] to \"./platform\"\n\nmain =\n"); @@ -16,12 +20,11 @@ fn promote_expr_to_module(src: &str) -> String { } #[allow(dead_code)] -pub fn helper_wasm<'a>( +pub fn helper_wasm<'a, T: Wasm32TestResult>( arena: &'a bumpalo::Bump, src: &str, stdlib: &'a roc_builtins::std::StdLib, - _is_gen_test: bool, - _ignore_problems: bool, + _result_type_dummy: &T, ) -> wasmer::Instance { use std::path::{Path, PathBuf}; @@ -91,7 +94,11 @@ pub fn helper_wasm<'a>( exposed_to_host, }; - let module_bytes = roc_gen_wasm::build_module(&env, procedures).unwrap(); + let (mut builder, main_function_index) = + roc_gen_wasm::build_module_help(&env, procedures).unwrap(); + T::insert_test_wrapper(&mut builder, TEST_WRAPPER_NAME, main_function_index); + + let module_bytes = builder.build().to_bytes().unwrap(); // for debugging (e.g. with wasm2wat) if false { @@ -128,44 +135,40 @@ pub fn helper_wasm<'a>( } #[allow(dead_code)] -pub fn assert_wasm_evals_to_help(src: &str, ignore_problems: bool) -> Result +pub fn assert_wasm_evals_to_help(src: &str, expected: T) -> Result where - T: Copy, + T: FromWasm32Memory + Wasm32TestResult, { let arena = bumpalo::Bump::new(); // NOTE the stdlib must be in the arena; just taking a reference will segfault let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); - let is_gen_test = true; - let instance = - crate::helpers::eval::helper_wasm(&arena, src, stdlib, is_gen_test, ignore_problems); + let instance = crate::helpers::eval::helper_wasm(&arena, src, stdlib, &expected); - let main_function = instance.exports.get_function("#UserApp_main_1").unwrap(); + let memory = instance.exports.get_memory("memory").unwrap(); - match main_function.call(&[]) { + let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap(); + + match test_wrapper.call(&[]) { Err(e) => Err(format!("{:?}", e)), Ok(result) => { - let integer = match result[0] { - wasmer::Value::I64(a) => a, - wasmer::Value::F64(a) => a.to_bits() as i64, + let address = match result[0] { + wasmer::Value::I32(a) => a, _ => panic!(), }; - let output_ptr: &T; - unsafe { - output_ptr = std::mem::transmute::<&i64, &T>(&integer); - } + let output = ::decode(memory, address as u32); - Ok(*output_ptr) + Ok(output) } } } #[macro_export] macro_rules! assert_wasm_evals_to { - ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => { - match $crate::helpers::eval::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems) { + ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { + match $crate::helpers::eval::assert_wasm_evals_to_help::<$ty>($src, $expected) { Err(msg) => println!("{:?}", msg), Ok(actual) => { #[allow(clippy::bool_assert_comparison)] @@ -175,11 +178,11 @@ macro_rules! assert_wasm_evals_to { }; ($src:expr, $expected:expr, $ty:ty) => { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity); }; ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform); }; } @@ -191,7 +194,7 @@ macro_rules! assert_evals_to { ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { // Same as above, except with an additional transformation argument. { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform); } }; } diff --git a/compiler/gen_wasm/tests/helpers/mod.rs b/compiler/gen_wasm/tests/helpers/mod.rs index 7e538193f7..c28fee53d9 100644 --- a/compiler/gen_wasm/tests/helpers/mod.rs +++ b/compiler/gen_wasm/tests/helpers/mod.rs @@ -2,6 +2,7 @@ extern crate bumpalo; #[macro_use] pub mod eval; +pub mod wasm32_test_result; /// Used in the with_larger_debug_stack() function, for tests that otherwise /// run out of stack space in debug builds (but don't in --release builds) diff --git a/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs b/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs new file mode 100644 index 0000000000..184edf43f7 --- /dev/null +++ b/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs @@ -0,0 +1,165 @@ +use parity_wasm::builder; +use parity_wasm::builder::ModuleBuilder; +use parity_wasm::elements::{Instruction, Instruction::*, Instructions, Internal, ValueType}; + +use roc_gen_wasm::from_wasm32_memory::FromWasm32Memory; +use roc_gen_wasm::*; +use roc_std::{RocDec, RocList, RocOrder, RocStr}; + +pub trait Wasm32TestResult { + fn insert_test_wrapper( + module_builder: &mut ModuleBuilder, + wrapper_name: &str, + main_function_index: u32, + ) { + let instructions = Self::build_wrapper_body(main_function_index); + + let signature = builder::signature().with_result(ValueType::I32).build_sig(); + + let function_def = builder::function() + .with_signature(signature) + .body() + .with_instructions(Instructions::new(instructions)) + .build() // body + .build(); // function + + let location = module_builder.push_function(function_def); + let export = builder::export() + .field(wrapper_name) + .with_internal(Internal::Function(location.body)) + .build(); + + module_builder.push_export(export); + } + + fn build_wrapper_body(main_function_index: u32) -> Vec; +} + +fn build_wrapper_body_prelude(stack_memory_size: usize) -> Vec { + vec![ + GetGlobal(STACK_POINTER_GLOBAL_ID), + I32Const(stack_memory_size as i32), + I32Sub, + SetGlobal(STACK_POINTER_GLOBAL_ID), + ] +} + +macro_rules! build_wrapper_body_primitive { + ($store_instruction: expr, $align: expr) => { + fn build_wrapper_body(main_function_index: u32) -> Vec { + const MAX_ALIGNED_SIZE: usize = 16; + let mut instructions = build_wrapper_body_prelude(MAX_ALIGNED_SIZE); + instructions.extend([ + GetGlobal(STACK_POINTER_GLOBAL_ID), + // + // Call the main function with no arguments. Get primitive back. + Call(main_function_index), + // + // Store the primitive at the allocated address + $store_instruction($align, 0), + // + // Return the result pointer + GetGlobal(STACK_POINTER_GLOBAL_ID), + End, + ]); + instructions + } + }; +} + +macro_rules! wasm_test_result_primitive { + ($type_name: ident, $store_instruction: expr, $align: expr) => { + impl Wasm32TestResult for $type_name { + build_wrapper_body_primitive!($store_instruction, $align); + } + }; +} + +fn build_wrapper_body_stack_memory(main_function_index: u32, size: usize) -> Vec { + let mut instructions = build_wrapper_body_prelude(size); + instructions.extend([ + // + // Call the main function with the allocated address to write the result. + // No value is returned to the VM stack. This is the same as in compiled C. + GetGlobal(STACK_POINTER_GLOBAL_ID), + Call(main_function_index), + // + // Return the result address + GetGlobal(STACK_POINTER_GLOBAL_ID), + End, + ]); + instructions +} + +macro_rules! wasm_test_result_stack_memory { + ($type_name: ident) => { + impl Wasm32TestResult for $type_name { + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, $type_name::ACTUAL_WIDTH) + } + } + }; +} + +wasm_test_result_primitive!(bool, I32Store8, ALIGN_1); +wasm_test_result_primitive!(RocOrder, I32Store8, ALIGN_1); + +wasm_test_result_primitive!(u8, I32Store8, ALIGN_1); +wasm_test_result_primitive!(i8, I32Store8, ALIGN_1); +wasm_test_result_primitive!(u16, I32Store16, ALIGN_2); +wasm_test_result_primitive!(i16, I32Store16, ALIGN_2); +wasm_test_result_primitive!(u32, I32Store, ALIGN_4); +wasm_test_result_primitive!(i32, I32Store, ALIGN_4); +wasm_test_result_primitive!(u64, I64Store, ALIGN_8); +wasm_test_result_primitive!(i64, I64Store, ALIGN_8); + +wasm_test_result_primitive!(f32, F32Store, ALIGN_8); +wasm_test_result_primitive!(f64, F64Store, ALIGN_8); + +wasm_test_result_stack_memory!(u128); +wasm_test_result_stack_memory!(i128); +wasm_test_result_stack_memory!(RocDec); +wasm_test_result_stack_memory!(RocStr); + +impl Wasm32TestResult for RocList { + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, 12) + } +} + +impl Wasm32TestResult for &'_ T { + build_wrapper_body_primitive!(I32Store, ALIGN_4); +} + +impl Wasm32TestResult for [T; N] +where + T: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, N * T::ACTUAL_WIDTH) + } +} + +impl Wasm32TestResult for (T, U) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, T::ACTUAL_WIDTH + U::ACTUAL_WIDTH) + } +} + +impl Wasm32TestResult for (T, U, V) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + U::ACTUAL_WIDTH + V::ACTUAL_WIDTH, + ) + } +} diff --git a/compiler/gen_wasm/tests/wasm_num.rs b/compiler/gen_wasm/tests/wasm_num.rs index 57ef682f76..bc327aa0eb 100644 --- a/compiler/gen_wasm/tests/wasm_num.rs +++ b/compiler/gen_wasm/tests/wasm_num.rs @@ -11,7 +11,7 @@ extern crate libc; mod helpers; #[cfg(all(test, any(target_os = "linux", target_os = "macos"), any(target_arch = "x86_64"/*, target_arch = "aarch64"*/)))] -mod dev_num { +mod wasm_num { #[test] fn i64_values() { assert_evals_to!("0", 0, i64); @@ -36,6 +36,101 @@ mod dev_num { assert_evals_to!(&format!("{:0.1}", f64::MAX), f64::MAX, f64); } + #[test] + fn i8_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I8 + x = 0x7f + 0x7f + + x + "# + ), + -2, + i8 + ); + } + + #[test] + fn i16_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I16 + x = 0x7fff + 0x7fff + + x + "# + ), + -2, + i16 + ); + } + + #[test] + fn i32_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I32 + x = 0x7fffffff + 0x7fffffff + + x + "# + ), + -2, + i32 + ); + } + + #[test] + fn u8_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U8 + x = 0xff + 0xff + + x + "# + ), + 0xfe, + u8 + ); + } + + #[test] + fn u16_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U16 + x = 0xffff + 0xffff + + x + "# + ), + 0xfffe, + u16 + ); + } + #[test] + fn u32_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U32 + x = 0xffffffff + 0xffffffff + + x + "# + ), + 0xfffffffe, + u32 + ); + } + #[test] fn gen_add_i64() { assert_evals_to!( @@ -49,44 +144,149 @@ mod dev_num { ); } - // #[test] - // fn gen_add_f64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1.1 + 2.4 + 3 - // "# - // ), - // 6.5, - // f64 - // ); - // } + #[test] + fn if_then_else() { + assert_evals_to!( + indoc!( + r#" + cond : Bool + cond = True - // #[test] - // fn gen_sub_i64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 - 2 - 3 - // "# - // ), - // -4, - // i64 - // ); - // } + if cond then + 0 + else + 1 + "# + ), + 0, + i64 + ); + } - // #[test] - // fn gen_mul_i64() { - // assert_evals_to!( - // indoc!( - // r#" - // 2 * 4 * 6 - // "# - // ), - // 48, - // i64 - // ); - // } + #[test] + fn rgb_red() { + assert_evals_to!( + indoc!( + r#" + when Red is + Red -> 111 + Green -> 222 + Blue -> 333 + "# + ), + 111, + i64 + ); + } + + #[test] + fn rgb_green() { + assert_evals_to!( + indoc!( + r#" + when Green is + Red -> 111 + Green -> 222 + Blue -> 333 + "# + ), + 222, + i64 + ); + } + + #[test] + fn rgb_blue() { + assert_evals_to!( + indoc!( + r#" + when Blue is + Red -> 111 + Green -> 222 + Blue -> 333 + "# + ), + 333, + i64 + ); + } + + #[test] + fn join_point() { + assert_evals_to!( + indoc!( + r#" + x = if True then 111 else 222 + + x + 123 + "# + ), + 234, + i64 + ); + } + + #[test] + fn factorial() { + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + fac : I32, I32 -> I32 + fac = \n, accum -> + if n > 1 then + fac (n - 1) (n * accum) + else + accum + + main : I32 + main = fac 8 1 + "# + ), + 40_320, + i32 + ); + } + + #[test] + fn gen_add_f64() { + assert_evals_to!( + indoc!( + r#" + 1.1 + 2.4 + 3 + "# + ), + 6.5, + f64 + ); + } + + #[test] + fn gen_sub_i64() { + assert_evals_to!( + indoc!( + r#" + 1 - 2 - 3 + "# + ), + -4, + i64 + ); + } + + #[test] + fn gen_mul_i64() { + assert_evals_to!( + indoc!( + r#" + 2 * 4 * 6 + "# + ), + 48, + i64 + ); + } #[test] fn i64_force_stack() { @@ -371,18 +571,18 @@ mod dev_num { // ); // } - // #[test] - // fn gen_sub_f64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1.5 - 2.4 - 3 - // "# - // ), - // -3.9, - // f64 - // ); - // } + #[test] + fn gen_sub_f64() { + assert_evals_to!( + indoc!( + r#" + 1.5 - 2.4 - 3 + "# + ), + -3.9, + f64 + ); + } // #[test] // fn gen_div_i64() { @@ -580,31 +780,31 @@ mod dev_num { // assert_evals_to!("0.0 >= 0.0", true, bool); // } - // #[test] - // fn gen_order_of_arithmetic_ops() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 + 3 * 7 - 2 - // "# - // ), - // 20, - // i64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops() { + assert_evals_to!( + indoc!( + r#" + 1 + 3 * 7 - 2 + "# + ), + 20, + i64 + ); + } - // #[test] - // fn gen_order_of_arithmetic_ops_complex_float() { - // assert_evals_to!( - // indoc!( - // r#" - // 3 - 48 * 2.0 - // "# - // ), - // -93.0, - // f64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops_complex_float() { + assert_evals_to!( + indoc!( + r#" + 3 - 48 * 2.0 + "# + ), + -93.0, + f64 + ); + } // #[test] // fn if_guard_bind_variable_false() { diff --git a/compiler/gen_wasm/tests/wasm_records.rs b/compiler/gen_wasm/tests/wasm_records.rs index 63f72357a2..9c776ecfaa 100644 --- a/compiler/gen_wasm/tests/wasm_records.rs +++ b/compiler/gen_wasm/tests/wasm_records.rs @@ -5,7 +5,7 @@ extern crate indoc; mod helpers; #[cfg(all(test, target_os = "linux", any(target_arch = "x86_64"/*, target_arch = "aarch64"*/)))] -mod dev_records { +mod wasm_records { // #[test] // fn basic_record() { // assert_evals_to!( @@ -389,18 +389,18 @@ mod dev_records { // // ); // // } - // #[test] - // fn i64_record1_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3 } - // "# - // ), - // 3, - // i64 - // ); - // } + #[test] + fn i64_record1_literal() { + assert_evals_to!( + indoc!( + r#" + { a: 3 } + "# + ), + 3, + i64 + ); + } // // #[test] // // fn i64_record9_literal() { @@ -428,21 +428,21 @@ mod dev_records { // // ); // // } - // #[test] - // fn bool_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // x : Bool - // x = True + #[test] + fn bool_literal() { + assert_evals_to!( + indoc!( + r#" + x : Bool + x = True - // x - // "# - // ), - // true, - // bool - // ); - // } + x + "# + ), + true, + bool + ); + } // #[test] // fn optional_field_when_use_default() { diff --git a/compiler/load/src/file.rs b/compiler/load/src/file.rs index 1c8f43612b..49b0a0c291 100644 --- a/compiler/load/src/file.rs +++ b/compiler/load/src/file.rs @@ -723,7 +723,21 @@ pub struct MonomorphizedModule<'a> { impl<'a> MonomorphizedModule<'a> { pub fn total_problems(&self) -> usize { - self.can_problems.len() + self.type_problems.len() + self.mono_problems.len() + let mut total = 0; + + for problems in self.can_problems.values() { + total += problems.len(); + } + + for problems in self.type_problems.values() { + total += problems.len(); + } + + for problems in self.mono_problems.values() { + total += problems.len(); + } + + total } } @@ -2099,8 +2113,6 @@ fn update<'a>( &mut state.procedures, ); - Proc::insert_refcount_operations(arena, &mut state.procedures); - // display the mono IR of the module, for debug purposes if roc_mono::ir::PRETTY_PRINT_IR_SYMBOLS { let procs_string = state @@ -2114,6 +2126,8 @@ fn update<'a>( println!("{}", result); } + Proc::insert_refcount_operations(arena, &mut state.procedures); + // This is not safe with the new non-recursive RC updates that we do for tag unions // // Proc::optimize_refcount_operations( @@ -3927,7 +3941,7 @@ fn make_specializations<'a>( ); let external_specializations_requested = procs.externals_we_need.clone(); - let procedures = procs.get_specialized_procs_without_rc(mono_env.arena); + let procedures = procs.get_specialized_procs_without_rc(&mut mono_env); let make_specializations_end = SystemTime::now(); module_timing.make_specializations = make_specializations_end diff --git a/compiler/mono/src/alias_analysis.rs b/compiler/mono/src/alias_analysis.rs index 229ed7cbee..f8e5dddb71 100644 --- a/compiler/mono/src/alias_analysis.rs +++ b/compiler/mono/src/alias_analysis.rs @@ -589,9 +589,9 @@ fn call_spec( let index = builder.add_make_tuple(block, &[])?; let argument = if closure_env_layout.is_none() { - builder.add_make_tuple(block, &[first, index])? + builder.add_make_tuple(block, &[index, first])? } else { - builder.add_make_tuple(block, &[first, index, closure_env])? + builder.add_make_tuple(block, &[index, first, closure_env])? }; builder.add_call(block, spec_var, module, name, argument)?; } @@ -1191,6 +1191,11 @@ fn layout_spec_help( match layout { Builtin(builtin) => builtin_spec(builder, builtin, when_recursive), Struct(fields) => build_recursive_tuple_type(builder, fields, when_recursive), + LambdaSet(lambda_set) => layout_spec_help( + builder, + &lambda_set.runtime_representation(), + when_recursive, + ), Union(union_layout) => { let variant_types = build_variant_types(builder, union_layout)?; @@ -1236,7 +1241,7 @@ fn builtin_spec( match builtin { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize => builder.add_tuple_type(&[]), - Decimal | Float128 | Float64 | Float32 | Float16 => builder.add_tuple_type(&[]), + Decimal | Float128 | Float64 | Float32 => builder.add_tuple_type(&[]), Str | EmptyStr => str_type(builder), Dict(key_layout, value_layout) => { let value_type = layout_spec_help(builder, value_layout, when_recursive)?; diff --git a/compiler/mono/src/decision_tree.rs b/compiler/mono/src/decision_tree.rs index 436eb43446..16b3ea0a84 100644 --- a/compiler/mono/src/decision_tree.rs +++ b/compiler/mono/src/decision_tree.rs @@ -1,6 +1,7 @@ use crate::exhaustive::{Ctor, RenderAs, TagId, Union}; use crate::ir::{ - BranchInfo, DestructType, Env, Expr, JoinPointId, Literal, Param, Pattern, Procs, Stmt, + BranchInfo, DestructType, Env, Expr, FloatPrecision, IntPrecision, JoinPointId, Literal, Param, + Pattern, Procs, Stmt, }; use crate::layout::{Builtin, Layout, LayoutCache, UnionLayout}; use roc_collections::all::{MutMap, MutSet}; @@ -85,8 +86,8 @@ enum Test<'a> { union: crate::exhaustive::Union, arguments: Vec<(Pattern<'a>, Layout<'a>)>, }, - IsInt(i128), - IsFloat(u64), + IsInt(i128, IntPrecision), + IsFloat(u64, FloatPrecision), IsDecimal(RocDec), IsStr(Box), IsBit(bool), @@ -95,6 +96,7 @@ enum Test<'a> { num_alts: usize, }, } + use std::hash::{Hash, Hasher}; impl<'a> Hash for Test<'a> { fn hash(&self, state: &mut H) { @@ -106,13 +108,15 @@ impl<'a> Hash for Test<'a> { tag_id.hash(state); // The point of this custom implementation is to not hash the tag arguments } - IsInt(v) => { + IsInt(v, width) => { state.write_u8(1); v.hash(state); + width.hash(state); } - IsFloat(v) => { + IsFloat(v, width) => { state.write_u8(2); v.hash(state); + width.hash(state); } IsStr(v) => { state.write_u8(3); @@ -306,8 +310,8 @@ fn tests_are_complete_help(last_test: &Test, number_of_tests: usize) -> bool { Test::IsCtor { union, .. } => number_of_tests == union.alternatives.len(), Test::IsByte { num_alts, .. } => number_of_tests == *num_alts, Test::IsBit(_) => number_of_tests == 2, - Test::IsInt(_) => false, - Test::IsFloat(_) => false, + Test::IsInt(_, _) => false, + Test::IsFloat(_, _) => false, Test::IsDecimal(_) => false, Test::IsStr(_) => false, } @@ -561,8 +565,8 @@ fn test_at_path<'a>( tag_id: *tag_id, num_alts: union.alternatives.len(), }, - IntLiteral(v) => IsInt(*v), - FloatLiteral(v) => IsFloat(*v), + IntLiteral(v, precision) => IsInt(*v, *precision), + FloatLiteral(v, precision) => IsFloat(*v, *precision), DecimalLiteral(v) => IsDecimal(*v), StrLiteral(v) => IsStr(v.clone()), }; @@ -807,8 +811,9 @@ fn to_relevant_branch_help<'a>( _ => None, }, - IntLiteral(int) => match test { - IsInt(is_int) if int == *is_int => { + IntLiteral(int, p1) => match test { + IsInt(is_int, p2) if int == *is_int => { + debug_assert_eq!(p1, *p2); start.extend(end); Some(Branch { goal: branch.goal, @@ -819,8 +824,9 @@ fn to_relevant_branch_help<'a>( _ => None, }, - FloatLiteral(float) => match test { - IsFloat(test_float) if float == *test_float => { + FloatLiteral(float, p1) => match test { + IsFloat(test_float, p2) if float == *test_float => { + debug_assert_eq!(p1, *p2); start.extend(end); Some(Branch { goal: branch.goal, @@ -928,8 +934,8 @@ fn needs_tests(pattern: &Pattern) -> bool { | AppliedTag { .. } | BitLiteral { .. } | EnumLiteral { .. } - | IntLiteral(_) - | FloatLiteral(_) + | IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | StrLiteral(_) => true, } @@ -1280,22 +1286,22 @@ fn test_to_equality<'a>( _ => unreachable!("{:?}", (cond_layout, union)), } } - Test::IsInt(test_int) => { + Test::IsInt(test_int, precision) => { // TODO don't downcast i128 here debug_assert!(test_int <= i64::MAX as i128); let lhs = Expr::Literal(Literal::Int(test_int as i128)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Int64), lhs)); + stores.push((lhs_symbol, precision.as_layout(), lhs)); (stores, lhs_symbol, rhs_symbol, None) } - Test::IsFloat(test_int) => { + Test::IsFloat(test_int, precision) => { // TODO maybe we can actually use i64 comparison here? let test_float = f64::from_bits(test_int as u64); let lhs = Expr::Literal(Literal::Float(test_float)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Float64), lhs)); + stores.push((lhs_symbol, precision.as_layout(), lhs)); (stores, lhs_symbol, rhs_symbol, None) } @@ -1303,7 +1309,7 @@ fn test_to_equality<'a>( Test::IsDecimal(test_dec) => { let lhs = Expr::Literal(Literal::Int(test_dec.0)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Int128), lhs)); + stores.push((lhs_symbol, *cond_layout, lhs)); (stores, lhs_symbol, rhs_symbol, None) } @@ -1737,8 +1743,8 @@ fn decide_to_branching<'a>( ); let tag = match test { - Test::IsInt(v) => v as u64, - Test::IsFloat(v) => v as u64, + Test::IsInt(v, _) => v as u64, + Test::IsFloat(v, _) => v as u64, Test::IsBit(v) => v as u64, Test::IsByte { tag_id, .. } => tag_id as u64, Test::IsCtor { tag_id, .. } => tag_id as u64, diff --git a/compiler/mono/src/exhaustive.rs b/compiler/mono/src/exhaustive.rs index 51b20d32aa..77c28d64a1 100644 --- a/compiler/mono/src/exhaustive.rs +++ b/compiler/mono/src/exhaustive.rs @@ -65,8 +65,8 @@ fn simplify(pattern: &crate::ir::Pattern) -> Pattern { use crate::ir::Pattern::*; match pattern { - IntLiteral(v) => Literal(Literal::Int(*v)), - FloatLiteral(v) => Literal(Literal::Float(*v)), + IntLiteral(v, _) => Literal(Literal::Int(*v)), + FloatLiteral(v, _) => Literal(Literal::Float(*v)), DecimalLiteral(v) => Literal(Literal::Decimal(*v)), StrLiteral(v) => Literal(Literal::Str(v.clone())), diff --git a/compiler/mono/src/ir.rs b/compiler/mono/src/ir.rs index b39bb8b722..bc620bdf74 100644 --- a/compiler/mono/src/ir.rs +++ b/compiler/mono/src/ir.rs @@ -54,6 +54,7 @@ macro_rules! return_on_layout_error_help { #[derive(Debug, Clone, Copy)] pub enum OptLevel { + Development, Normal, Optimize, } @@ -272,6 +273,33 @@ impl<'a> Proc<'a> { proc.body = b.clone(); } } + + fn make_tail_recursive(&mut self, env: &mut Env<'a, '_>) { + let mut args = Vec::with_capacity_in(self.args.len(), env.arena); + let mut proc_args = Vec::with_capacity_in(self.args.len(), env.arena); + + for (layout, symbol) in self.args { + let new = env.unique_symbol(); + args.push((*layout, *symbol, new)); + proc_args.push((*layout, new)); + } + + use self::SelfRecursive::*; + if let SelfRecursive(id) = self.is_self_recursive { + let transformed = crate::tail_recursion::make_tail_recursive( + env.arena, + id, + self.name, + self.body.clone(), + args.into_bump_slice(), + ); + + if let Some(with_tco) = transformed { + self.body = with_tco; + self.args = proc_args.into_bump_slice(); + } + } + } } #[derive(Clone, Debug)] @@ -350,7 +378,7 @@ pub enum InProgressProc<'a> { impl<'a> Procs<'a> { pub fn get_specialized_procs_without_rc( self, - arena: &'a Bump, + env: &mut Env<'a, '_>, ) -> MutMap<(Symbol, ProcLayout<'a>), Proc<'a>> { let mut result = MutMap::with_capacity_and_hasher(self.specialized.len(), default_hasher()); @@ -376,16 +404,7 @@ impl<'a> Procs<'a> { panic!(); } Done(mut proc) => { - use self::SelfRecursive::*; - if let SelfRecursive(id) = proc.is_self_recursive { - proc.body = crate::tail_recursion::make_tail_recursive( - arena, - id, - proc.name, - proc.body.clone(), - proc.args, - ); - } + proc.make_tail_recursive(env); result.insert(key, proc); } @@ -395,86 +414,6 @@ impl<'a> Procs<'a> { result } - // TODO investigate make this an iterator? - pub fn get_specialized_procs( - self, - arena: &'a Bump, - ) -> MutMap<(Symbol, ProcLayout<'a>), Proc<'a>> { - let mut result = MutMap::with_capacity_and_hasher(self.specialized.len(), default_hasher()); - - for ((s, toplevel), in_prog_proc) in self.specialized.into_iter() { - match in_prog_proc { - InProgress => unreachable!( - "The procedure {:?} should have be done by now", - (s, toplevel) - ), - Done(proc) => { - result.insert((s, toplevel), proc); - } - } - } - - for (_, proc) in result.iter_mut() { - use self::SelfRecursive::*; - if let SelfRecursive(id) = proc.is_self_recursive { - proc.body = crate::tail_recursion::make_tail_recursive( - arena, - id, - proc.name, - proc.body.clone(), - proc.args, - ); - } - } - - let borrow_params = arena.alloc(crate::borrow::infer_borrow(arena, &result)); - - crate::inc_dec::visit_procs(arena, borrow_params, &mut result); - - result - } - - pub fn get_specialized_procs_help( - self, - arena: &'a Bump, - ) -> ( - MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>, - &'a crate::borrow::ParamMap<'a>, - ) { - let mut result = MutMap::with_capacity_and_hasher(self.specialized.len(), default_hasher()); - - for ((s, toplevel), in_prog_proc) in self.specialized.into_iter() { - match in_prog_proc { - InProgress => unreachable!( - "The procedure {:?} should have be done by now", - (s, toplevel) - ), - Done(proc) => { - result.insert((s, toplevel), proc); - } - } - } - - for (_, proc) in result.iter_mut() { - use self::SelfRecursive::*; - if let SelfRecursive(id) = proc.is_self_recursive { - proc.body = crate::tail_recursion::make_tail_recursive( - arena, - id, - proc.name, - proc.body.clone(), - proc.args, - ); - } - } - - let borrow_params = arena.alloc(crate::borrow::infer_borrow(arena, &result)); - - crate::inc_dec::visit_procs(arena, borrow_params, &mut result); - - (result, borrow_params) - } - // TODO trim down these arguments! #[allow(clippy::too_many_arguments)] pub fn insert_named( @@ -756,7 +695,20 @@ impl<'a> Procs<'a> { // the `layout` is a function pointer, while `_ignore_layout` can be a // closure. We only specialize functions, storing this value with a closure // layout will give trouble. - self.specialized.insert((symbol, layout), Done(proc)); + let arguments = + Vec::from_iter_in(proc.args.iter().map(|(l, _)| *l), env.arena) + .into_bump_slice(); + + let proper_layout = ProcLayout { + arguments, + result: proc.ret_layout, + }; + + // NOTE: some function are specialized to have a closure, but don't actually + // need any closure argument. Here is where we correct this sort of thing, + // by trusting the layout of the Proc, not of what we specialize for + self.specialized.remove(&(symbol, layout)); + self.specialized.insert((symbol, proper_layout), Done(proc)); } Err(error) => { panic!("TODO generate a RuntimeError message for {:?}", error); @@ -1905,7 +1857,7 @@ fn generate_runtime_error_function<'a>( args.push((*arg, env.unique_symbol())); } - args.push((lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)); + args.push((Layout::LambdaSet(lambda_set), Symbol::ARG_CLOSURE)); (args.into_bump_slice(), *ret_layout) } @@ -1981,7 +1933,29 @@ fn specialize_external<'a>( match layout { RawFunctionLayout::Function(argument_layouts, lambda_set, return_layout) => { let assigned = env.unique_symbol(); - let unit = env.unique_symbol(); + + let mut argument_symbols = + Vec::with_capacity_in(argument_layouts.len(), env.arena); + let mut proc_arguments = + Vec::with_capacity_in(argument_layouts.len() + 1, env.arena); + let mut top_level_arguments = + Vec::with_capacity_in(argument_layouts.len() + 1, env.arena); + + for layout in argument_layouts { + let symbol = env.unique_symbol(); + + proc_arguments.push((*layout, symbol)); + + argument_symbols.push(symbol); + top_level_arguments.push(*layout); + } + + // the proc needs to take an extra closure argument + let lambda_set_layout = Layout::LambdaSet(lambda_set); + proc_arguments.push((lambda_set_layout, Symbol::ARG_CLOSURE)); + + // this should also be reflected in the TopLevel signature + top_level_arguments.push(lambda_set_layout); let hole = env.arena.alloc(Stmt::Ret(assigned)); @@ -1989,20 +1963,16 @@ fn specialize_external<'a>( env, lambda_set, Symbol::ARG_CLOSURE, - env.arena.alloc([unit]), + argument_symbols.into_bump_slice(), argument_layouts, *return_layout, assigned, hole, ); - let body = let_empty_struct(unit, env.arena.alloc(body)); - let proc = Proc { name, - args: env - .arena - .alloc([(lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)]), + args: proc_arguments.into_bump_slice(), body, closure_data_layout: None, ret_layout: *return_layout, @@ -2013,7 +1983,7 @@ fn specialize_external<'a>( let top_level = ProcLayout::new( env.arena, - env.arena.alloc([lambda_set.runtime_representation()]), + top_level_arguments.into_bump_slice(), *return_layout, ); @@ -2061,7 +2031,7 @@ fn specialize_external<'a>( env.subs.rollback_to(snapshot); let closure_data_layout = match opt_closure_layout { - Some(closure_layout) => closure_layout.runtime_representation(), + Some(lambda_set) => Layout::LambdaSet(lambda_set), None => Layout::Struct(&[]), }; @@ -2211,7 +2181,7 @@ fn specialize_external<'a>( env.subs.rollback_to(snapshot); let closure_data_layout = match opt_closure_layout { - Some(closure_layout) => Some(closure_layout.runtime_representation()), + Some(lambda_set) => Some(Layout::LambdaSet(lambda_set)), None => None, }; @@ -2322,7 +2292,7 @@ fn build_specialized_proc<'a>( Some(lambda_set) if pattern_symbols.last() == Some(&Symbol::ARG_CLOSURE) => { // here we define the lifted (now top-level) f function. Its final argument is `Symbol::ARG_CLOSURE`, // it stores the closure structure (just an integer in this case) - proc_args.push((lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)); + proc_args.push((Layout::LambdaSet(lambda_set), Symbol::ARG_CLOSURE)); debug_assert_eq!( pattern_layouts_len + 1, @@ -2359,7 +2329,7 @@ fn build_specialized_proc<'a>( } Ordering::Greater => { if pattern_symbols.is_empty() { - let ret_layout = lambda_set.runtime_representation(); + let ret_layout = Layout::LambdaSet(lambda_set); Ok(FunctionPointerBody { closure: None, ret_layout, @@ -2544,7 +2514,7 @@ where let raw = if procs.module_thunks.contains(&proc_name) { match raw { RawFunctionLayout::Function(_, lambda_set, _) => { - RawFunctionLayout::ZeroArgumentThunk(lambda_set.runtime_representation()) + RawFunctionLayout::ZeroArgumentThunk(Layout::LambdaSet(lambda_set)) } _ => raw, } @@ -2718,6 +2688,7 @@ macro_rules! match_on_closure_argument { let ret_layout = top_level.result; + match closure_data_layout { RawFunctionLayout::Function(_, lambda_set, _) => { lowlevel_match_on_lambda_set( @@ -2822,13 +2793,13 @@ pub fn with_hole<'a>( IntOrFloat::SignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(int)), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::UnsignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(int)), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), _ => unreachable!("unexpected float precision for integer"), @@ -2840,7 +2811,7 @@ pub fn with_hole<'a>( IntOrFloat::BinaryFloatType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Float(float)), - Layout::Builtin(float_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::DecimalFloatType => { @@ -2872,19 +2843,19 @@ pub fn with_hole<'a>( IntOrFloat::SignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(num.into())), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::UnsignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(num.into())), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::BinaryFloatType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Float(num as f64)), - Layout::Builtin(float_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::DecimalFloatType => { @@ -3748,13 +3719,15 @@ pub fn with_hole<'a>( match what_to_do { UpdateExisting(field) => { + substitute_in_exprs(env.arena, &mut stmt, assigned, symbols[0]); + stmt = assign_to_symbol( env, procs, layout_cache, field.var, *field.loc_expr.clone(), - assigned, + symbols[0], stmt, ); } @@ -4191,6 +4164,8 @@ fn construct_closure_data<'a>( assigned: Symbol, hole: &'a Stmt<'a>, ) -> Stmt<'a> { + let lambda_set_layout = Layout::LambdaSet(lambda_set); + match lambda_set.layout_for_member(name) { ClosureRepresentation::Union { tag_id, @@ -4222,12 +4197,7 @@ fn construct_closure_data<'a>( arguments: symbols, }; - Stmt::Let( - assigned, - expr, - lambda_set.runtime_representation(), - env.arena.alloc(hole), - ) + Stmt::Let(assigned, expr, lambda_set_layout, env.arena.alloc(hole)) } ClosureRepresentation::AlphabeticOrderStruct(field_layouts) => { debug_assert_eq!(field_layouts.len(), symbols.len()); @@ -4258,7 +4228,7 @@ fn construct_closure_data<'a>( let expr = Expr::Struct(symbols); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } ClosureRepresentation::Other(Layout::Builtin(Builtin::Int1)) => { debug_assert_eq!(symbols.len(), 0); @@ -4267,7 +4237,7 @@ fn construct_closure_data<'a>( let tag_id = name != lambda_set.set[0].0; let expr = Expr::Literal(Literal::Bool(tag_id)); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } ClosureRepresentation::Other(Layout::Builtin(Builtin::Int8)) => { debug_assert_eq!(symbols.len(), 0); @@ -4276,7 +4246,7 @@ fn construct_closure_data<'a>( let tag_id = lambda_set.set.iter().position(|(s, _)| *s == name).unwrap() as u8; let expr = Expr::Literal(Literal::Byte(tag_id)); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } _ => unreachable!(), } @@ -5683,8 +5653,8 @@ fn store_pattern_help<'a>( // do nothing return StorePattern::NotProductive(stmt); } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5818,8 +5788,8 @@ fn store_tag_pattern<'a>( Underscore => { // ignore } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5894,8 +5864,8 @@ fn store_newtype_pattern<'a>( Underscore => { // ignore } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5970,8 +5940,8 @@ fn store_record_destruct<'a>( // internally. But `y` is never used, so we must make sure it't not stored/loaded. return StorePattern::NotProductive(stmt); } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -6120,7 +6090,7 @@ fn reuse_function_symbol<'a>( let layout = match raw { RawFunctionLayout::ZeroArgumentThunk(layout) => layout, RawFunctionLayout::Function(_, lambda_set, _) => { - lambda_set.runtime_representation() + Layout::LambdaSet(lambda_set) } }; @@ -6218,7 +6188,7 @@ fn reuse_function_symbol<'a>( // TODO suspicious // let layout = Layout::Closure(argument_layouts, lambda_set, ret_layout); // panic!("suspicious"); - let layout = lambda_set.runtime_representation(); + let layout = Layout::LambdaSet(lambda_set); let top_level = ProcLayout::new(env.arena, &[], layout); procs.insert_passed_by_name( env, @@ -6238,9 +6208,14 @@ fn reuse_function_symbol<'a>( layout_cache, ); - // a function name (non-closure) that is passed along - // it never has closure data, so we use the empty struct - return let_empty_struct(symbol, env.arena.alloc(result)); + construct_closure_data( + env, + lambda_set, + original, + &[], + symbol, + env.arena.alloc(result), + ) } } RawFunctionLayout::ZeroArgumentThunk(ret_layout) => { @@ -6408,7 +6383,7 @@ fn call_by_name<'a>( procs, fn_var, proc_name, - env.arena.alloc(lambda_set.runtime_representation()), + env.arena.alloc(Layout::LambdaSet(lambda_set)), layout_cache, assigned, hole, @@ -6452,7 +6427,7 @@ fn call_by_name<'a>( procs, fn_var, proc_name, - env.arena.alloc(lambda_set.runtime_representation()), + env.arena.alloc(Layout::LambdaSet(lambda_set)), layout_cache, closure_data_symbol, env.arena.alloc(result), @@ -6582,7 +6557,7 @@ fn call_by_name_help<'a>( force_thunk( env, proc_name, - lambda_set.runtime_representation(), + Layout::LambdaSet(lambda_set), assigned, hole, ) @@ -6896,13 +6871,7 @@ fn call_specialized_proc<'a>( arguments: field_symbols, }; - build_call( - env, - call, - assigned, - lambda_set.runtime_representation(), - hole, - ) + build_call(env, call, assigned, Layout::LambdaSet(lambda_set), hole) } RawFunctionLayout::ZeroArgumentThunk(_) => { unreachable!() @@ -6942,8 +6911,8 @@ fn call_specialized_proc<'a>( pub enum Pattern<'a> { Identifier(Symbol), Underscore, - IntLiteral(i128), - FloatLiteral(u64), + IntLiteral(i128, IntPrecision), + FloatLiteral(u64, FloatPrecision), DecimalLiteral(RocDec), BitLiteral { value: bool, @@ -7021,22 +6990,36 @@ fn from_can_pattern_help<'a>( match can_pattern { Underscore => Ok(Pattern::Underscore), Identifier(symbol) => Ok(Pattern::Identifier(*symbol)), - IntLiteral(_, _, int) => Ok(Pattern::IntLiteral(*int as i128)), + IntLiteral(var, _, int) => { + match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, false) { + IntOrFloat::SignedIntType(precision) | IntOrFloat::UnsignedIntType(precision) => { + Ok(Pattern::IntLiteral(*int as i128, precision)) + } + other => { + panic!( + "Invalid precision for int pattern: {:?} has {:?}", + can_pattern, other + ) + } + } + } FloatLiteral(var, float_str, float) => { // TODO: Can I reuse num_argument_to_int_or_float here if I pass in true? match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, true) { - IntOrFloat::SignedIntType(_) => { - panic!("Invalid percision for float literal = {:?}", var) + IntOrFloat::SignedIntType(_) | IntOrFloat::UnsignedIntType(_) => { + panic!("Invalid precision for float pattern {:?}", var) } - IntOrFloat::UnsignedIntType(_) => { - panic!("Invalid percision for float literal = {:?}", var) + IntOrFloat::BinaryFloatType(precision) => { + Ok(Pattern::FloatLiteral(f64::to_bits(*float), precision)) } - IntOrFloat::BinaryFloatType(_) => Ok(Pattern::FloatLiteral(f64::to_bits(*float))), IntOrFloat::DecimalFloatType => { let dec = match RocDec::from_str(float_str) { - Some(d) => d, - None => panic!("Invalid decimal for float literal = {}. TODO: Make this a nice, user-friendly error message", float_str), - }; + Some(d) => d, + None => panic!( + r"Invalid decimal for float literal = {}. TODO: Make this a nice, user-friendly error message", + float_str + ), + }; Ok(Pattern::DecimalLiteral(dec)) } } @@ -7053,9 +7036,15 @@ fn from_can_pattern_help<'a>( } NumLiteral(var, num_str, num) => { match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, false) { - IntOrFloat::SignedIntType(_) => Ok(Pattern::IntLiteral(*num as i128)), - IntOrFloat::UnsignedIntType(_) => Ok(Pattern::IntLiteral(*num as i128)), - IntOrFloat::BinaryFloatType(_) => Ok(Pattern::FloatLiteral(*num as u64)), + IntOrFloat::SignedIntType(precision) => { + Ok(Pattern::IntLiteral(*num as i128, precision)) + } + IntOrFloat::UnsignedIntType(precision) => { + Ok(Pattern::IntLiteral(*num as i128, precision)) + } + IntOrFloat::BinaryFloatType(precision) => { + Ok(Pattern::FloatLiteral(*num as u64, precision)) + } IntOrFloat::DecimalFloatType => { let dec = match RocDec::from_str(num_str) { Some(d) => d, @@ -7637,7 +7626,7 @@ fn from_can_record_destruct<'a>( }) } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum IntPrecision { Usize, I128, @@ -7647,11 +7636,45 @@ pub enum IntPrecision { I8, } +impl IntPrecision { + pub fn as_layout(&self) -> Layout<'static> { + Layout::Builtin(self.as_builtin()) + } + + pub fn as_builtin(&self) -> Builtin<'static> { + use IntPrecision::*; + match self { + I128 => Builtin::Int128, + I64 => Builtin::Int64, + I32 => Builtin::Int32, + I16 => Builtin::Int16, + I8 => Builtin::Int8, + Usize => Builtin::Usize, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum FloatPrecision { F64, F32, } +impl FloatPrecision { + pub fn as_layout(&self) -> Layout<'static> { + Layout::Builtin(self.as_builtin()) + } + + pub fn as_builtin(&self) -> Builtin<'static> { + use FloatPrecision::*; + match self { + F64 => Builtin::Float64, + F32 => Builtin::Float32, + } + } +} + +#[derive(Debug)] pub enum IntOrFloat { SignedIntType(IntPrecision), UnsignedIntType(IntPrecision), @@ -7659,26 +7682,6 @@ pub enum IntOrFloat { DecimalFloatType, } -fn float_precision_to_builtin(precision: FloatPrecision) -> Builtin<'static> { - use FloatPrecision::*; - match precision { - F64 => Builtin::Float64, - F32 => Builtin::Float32, - } -} - -fn int_precision_to_builtin(precision: IntPrecision) -> Builtin<'static> { - use IntPrecision::*; - match precision { - I128 => Builtin::Int128, - I64 => Builtin::Int64, - I32 => Builtin::Int32, - I16 => Builtin::Int16, - I8 => Builtin::Int8, - Usize => Builtin::Usize, - } -} - /// Given the `a` in `Num a`, determines whether it's an int or a float pub fn num_argument_to_int_or_float( subs: &Subs, @@ -7975,8 +7978,8 @@ fn match_on_lambda_set<'a>( let result = union_lambda_set_to_switch( env, - lambda_set.set, - lambda_set.runtime_representation(), + lambda_set, + Layout::Union(union_layout), closure_tag_id_symbol, union_layout.tag_id_layout(), closure_data_symbol, @@ -8006,6 +8009,7 @@ fn match_on_lambda_set<'a>( union_lambda_set_branch_help( env, function_symbol, + lambda_set, closure_data_symbol, Layout::Struct(fields), argument_symbols, @@ -8054,7 +8058,7 @@ fn match_on_lambda_set<'a>( #[allow(clippy::too_many_arguments)] fn union_lambda_set_to_switch<'a>( env: &mut Env<'a, '_>, - lambda_set: &'a [(Symbol, &'a [Layout<'a>])], + lambda_set: LambdaSet<'a>, closure_layout: Layout<'a>, closure_tag_id_symbol: Symbol, closure_tag_id_layout: Layout<'a>, @@ -8065,7 +8069,7 @@ fn union_lambda_set_to_switch<'a>( assigned: Symbol, hole: &'a Stmt<'a>, ) -> Stmt<'a> { - if lambda_set.is_empty() { + if lambda_set.set.is_empty() { // NOTE this can happen if there is a type error somewhere. Since the lambda set is empty, // there is really nothing we can do here. We generate a runtime error here which allows // code gen to proceed. We then assume that we hit another (more descriptive) error before @@ -8077,11 +8081,12 @@ fn union_lambda_set_to_switch<'a>( let join_point_id = JoinPointId(env.unique_symbol()); - let mut branches = Vec::with_capacity_in(lambda_set.len(), env.arena); + let mut branches = Vec::with_capacity_in(lambda_set.set.len(), env.arena); - for (i, (function_symbol, _)) in lambda_set.iter().enumerate() { + for (i, (function_symbol, _)) in lambda_set.set.iter().enumerate() { let stmt = union_lambda_set_branch( env, + lambda_set, join_point_id, *function_symbol, closure_data_symbol, @@ -8124,6 +8129,7 @@ fn union_lambda_set_to_switch<'a>( #[allow(clippy::too_many_arguments)] fn union_lambda_set_branch<'a>( env: &mut Env<'a, '_>, + lambda_set: LambdaSet<'a>, join_point_id: JoinPointId, function_symbol: Symbol, closure_data_symbol: Symbol, @@ -8139,6 +8145,7 @@ fn union_lambda_set_branch<'a>( union_lambda_set_branch_help( env, function_symbol, + lambda_set, closure_data_symbol, closure_data_layout, argument_symbols_slice, @@ -8153,6 +8160,7 @@ fn union_lambda_set_branch<'a>( fn union_lambda_set_branch_help<'a>( env: &mut Env<'a, '_>, function_symbol: Symbol, + lambda_set: LambdaSet<'a>, closure_data_symbol: Symbol, closure_data_layout: Layout<'a>, argument_symbols_slice: &'a [Symbol], @@ -8165,12 +8173,19 @@ fn union_lambda_set_branch_help<'a>( Layout::Struct(&[]) | Layout::Builtin(Builtin::Int1) | Layout::Builtin(Builtin::Int8) => { (argument_layouts_slice, argument_symbols_slice) } + _ if lambda_set.member_does_not_need_closure_argument(function_symbol) => { + // sometimes unification causes a function that does not itself capture anything + // to still get a lambda set that does store information. We must not pass a closure + // argument in this case + + (argument_layouts_slice, argument_symbols_slice) + } _ => { // extend layouts with the layout of the closure environment let mut argument_layouts = Vec::with_capacity_in(argument_layouts_slice.len() + 1, env.arena); argument_layouts.extend(argument_layouts_slice); - argument_layouts.push(closure_data_layout); + argument_layouts.push(Layout::LambdaSet(lambda_set)); // extend symbols with the symbol of the closure environment let mut argument_symbols = diff --git a/compiler/mono/src/layout.rs b/compiler/mono/src/layout.rs index 3568ce5045..959d679cb3 100644 --- a/compiler/mono/src/layout.rs +++ b/compiler/mono/src/layout.rs @@ -189,6 +189,7 @@ pub enum Layout<'a> { /// this is important for closures that capture zero-sized values Struct(&'a [Layout<'a>]), Union(UnionLayout<'a>), + LambdaSet(LambdaSet<'a>), RecursivePointer, } @@ -454,6 +455,17 @@ impl<'a> LambdaSet<'a> { } } + pub fn member_does_not_need_closure_argument(&self, function_symbol: Symbol) -> bool { + match self.layout_for_member(function_symbol) { + ClosureRepresentation::Union { + alphabetic_order_fields, + .. + } => alphabetic_order_fields.is_empty(), + ClosureRepresentation::AlphabeticOrderStruct(fields) => fields.is_empty(), + ClosureRepresentation::Other(_) => false, + } + } + pub fn layout_for_member(&self, function_symbol: Symbol) -> ClosureRepresentation<'a> { debug_assert!( self.set.iter().any(|(s, _)| *s == function_symbol), @@ -531,7 +543,7 @@ impl<'a> LambdaSet<'a> { _ => { let mut arguments = Vec::with_capacity_in(argument_layouts.len() + 1, arena); arguments.extend(argument_layouts); - arguments.push(self.runtime_representation()); + arguments.push(Layout::LambdaSet(*self)); arguments.into_bump_slice() } @@ -606,7 +618,9 @@ impl<'a> LambdaSet<'a> { use UnionVariant::*; match variant { Never => Layout::Union(UnionLayout::NonRecursive(&[])), - Unit | UnitWithArguments | BoolUnion { .. } | ByteUnion(_) => { + BoolUnion { .. } => Layout::Builtin(Builtin::Int1), + ByteUnion { .. } => Layout::Builtin(Builtin::Int8), + Unit | UnitWithArguments => { // no useful information to store Layout::Struct(&[]) } @@ -667,7 +681,6 @@ pub enum Builtin<'a> { Float128, Float64, Float32, - Float16, Str, Dict(&'a Layout<'a>, &'a Layout<'a>), Set(&'a Layout<'a>), @@ -826,6 +839,7 @@ impl<'a> Layout<'a> { } } } + LambdaSet(lambda_set) => lambda_set.runtime_representation().safe_to_memcpy(), RecursivePointer => { // We cannot memcpy pointers, because then we would have the same pointer in multiple places! false @@ -890,6 +904,9 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => pointer_size, } } + LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .stack_size_without_alignment(pointer_size), RecursivePointer => pointer_size, } } @@ -919,6 +936,9 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => pointer_size, } } + Layout::LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .alignment_bytes(pointer_size), Layout::Builtin(builtin) => builtin.alignment_bytes(pointer_size), Layout::RecursivePointer => pointer_size, } @@ -929,6 +949,9 @@ impl<'a> Layout<'a> { Layout::Builtin(builtin) => builtin.allocation_alignment_bytes(pointer_size), Layout::Struct(_) => unreachable!("not heap-allocated"), Layout::Union(union_layout) => union_layout.allocation_alignment_bytes(pointer_size), + Layout::LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .allocation_alignment_bytes(pointer_size), Layout::RecursivePointer => unreachable!("should be looked up to get an actual layout"), } } @@ -979,6 +1002,7 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => true, } } + LambdaSet(lambda_set) => lambda_set.runtime_representation().contains_refcounted(), RecursivePointer => true, } } @@ -1002,6 +1026,7 @@ impl<'a> Layout<'a> { .append(alloc.text("}")) } Union(union_layout) => union_layout.to_doc(alloc, parens), + LambdaSet(lambda_set) => lambda_set.runtime_representation().to_doc(alloc, parens), RecursivePointer => alloc.text("*self"), } } @@ -1093,7 +1118,6 @@ impl<'a> Builtin<'a> { const F128_SIZE: u32 = 16; const F64_SIZE: u32 = std::mem::size_of::() as u32; const F32_SIZE: u32 = std::mem::size_of::() as u32; - const F16_SIZE: u32 = 2; /// Number of machine words in an empty one of these pub const STR_WORDS: u32 = 2; @@ -1123,7 +1147,6 @@ impl<'a> Builtin<'a> { Float128 => Builtin::F128_SIZE, Float64 => Builtin::F64_SIZE, Float32 => Builtin::F32_SIZE, - Float16 => Builtin::F16_SIZE, Str | EmptyStr => Builtin::STR_WORDS * pointer_size, Dict(_, _) | EmptyDict => Builtin::DICT_WORDS * pointer_size, Set(_) | EmptySet => Builtin::SET_WORDS * pointer_size, @@ -1150,7 +1173,6 @@ impl<'a> Builtin<'a> { Float128 => align_of::() as u32, Float64 => align_of::() as u32, Float32 => align_of::() as u32, - Float16 => align_of::() as u32, Dict(_, _) | EmptyDict => pointer_size, Set(_) | EmptySet => pointer_size, // we often treat these as i128 (64-bit systems) @@ -1158,8 +1180,8 @@ impl<'a> Builtin<'a> { // // In webassembly, For that to be safe // they must be aligned to allow such access - List(_) | EmptyList => pointer_size.max(8), - Str | EmptyStr => pointer_size.max(8), + List(_) | EmptyList => pointer_size, + Str | EmptyStr => pointer_size, } } @@ -1168,7 +1190,7 @@ impl<'a> Builtin<'a> { match self { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize | Decimal | Float128 | Float64 - | Float32 | Float16 | EmptyStr | EmptyDict | EmptyList | EmptySet => true, + | Float32 | EmptyStr | EmptyDict | EmptyList | EmptySet => true, Str | Dict(_, _) | Set(_) | List(_) => false, } } @@ -1179,7 +1201,7 @@ impl<'a> Builtin<'a> { match self { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize | Decimal | Float128 | Float64 - | Float32 | Float16 | EmptyStr | EmptyDict | EmptyList | EmptySet => false, + | Float32 | EmptyStr | EmptyDict | EmptyList | EmptySet => false, List(_) => true, Str | Dict(_, _) | Set(_) => true, @@ -1206,7 +1228,6 @@ impl<'a> Builtin<'a> { Float128 => alloc.text("Float128"), Float64 => alloc.text("Float64"), Float32 => alloc.text("Float32"), - Float16 => alloc.text("Float16"), EmptyStr => alloc.text("EmptyStr"), EmptyList => alloc.text("EmptyList"), @@ -1240,8 +1261,7 @@ impl<'a> Builtin<'a> { | Builtin::Decimal | Builtin::Float128 | Builtin::Float64 - | Builtin::Float32 - | Builtin::Float16 => unreachable!("not heap-allocated"), + | Builtin::Float32 => unreachable!("not heap-allocated"), Builtin::Str => pointer_size, Builtin::Dict(k, v) => k .alignment_bytes(pointer_size) @@ -1360,7 +1380,7 @@ fn layout_from_flat_type<'a>( Func(_, closure_var, _) => { let lambda_set = LambdaSet::from_var(env.arena, env.subs, closure_var, env.ptr_bytes)?; - Ok(lambda_set.runtime_representation()) + Ok(Layout::LambdaSet(lambda_set)) } Record(fields, ext_var) => { // extract any values from the ext_var diff --git a/compiler/mono/src/tail_recursion.rs b/compiler/mono/src/tail_recursion.rs index d0a75b0c2a..08dc545fc3 100644 --- a/compiler/mono/src/tail_recursion.rs +++ b/compiler/mono/src/tail_recursion.rs @@ -33,16 +33,16 @@ pub fn make_tail_recursive<'a>( id: JoinPointId, needle: Symbol, stmt: Stmt<'a>, - args: &'a [(Layout<'a>, Symbol)], -) -> Stmt<'a> { + args: &'a [(Layout<'a>, Symbol, Symbol)], +) -> Option> { let allocated = arena.alloc(stmt); match insert_jumps(arena, allocated, id, needle) { - None => allocated.clone(), + None => None, Some(new) => { // jumps were inserted, we must now add a join point let params = Vec::from_iter_in( - args.iter().map(|(layout, symbol)| Param { + args.iter().map(|(layout, symbol, _)| Param { symbol: *symbol, layout: *layout, borrow: true, @@ -52,16 +52,18 @@ pub fn make_tail_recursive<'a>( .into_bump_slice(); // TODO could this be &[]? - let args = Vec::from_iter_in(args.iter().map(|t| t.1), arena).into_bump_slice(); + let args = Vec::from_iter_in(args.iter().map(|t| t.2), arena).into_bump_slice(); let jump = arena.alloc(Stmt::Jump(id, args)); - Stmt::Join { + let join = Stmt::Join { id, remainder: jump, parameters: params, body: new, - } + }; + + Some(join) } } } diff --git a/compiler/reporting/src/error/canonicalize.rs b/compiler/reporting/src/error/canonicalize.rs index 69f1175a6e..13d3406593 100644 --- a/compiler/reporting/src/error/canonicalize.rs +++ b/compiler/reporting/src/error/canonicalize.rs @@ -1192,7 +1192,7 @@ fn not_found<'b>( alloc.reflow(" missing up-top"), ]); - let default_yes = alloc.reflow("these names seem close though:"); + let default_yes = alloc.reflow("Did you mean one of these?"); let to_details = |no_suggestion_details, yes_suggestion_details| { if suggestions.is_empty() { @@ -1240,7 +1240,7 @@ fn module_not_found<'b>( ]); let default_yes = alloc - .reflow("Is there an import missing? Perhaps there is a typo, these names seem close:"); + .reflow("Is there an import missing? Perhaps there is a typo. Did you mean one of these?"); let to_details = |no_suggestion_details, yes_suggestion_details| { if suggestions.is_empty() { diff --git a/compiler/reporting/src/error/type.rs b/compiler/reporting/src/error/type.rs index 115d3f6671..c4cd15d406 100644 --- a/compiler/reporting/src/error/type.rs +++ b/compiler/reporting/src/error/type.rs @@ -1,7 +1,8 @@ use roc_can::expected::{Expected, PExpected}; use roc_collections::all::{Index, MutSet, SendMap}; -use roc_module::ident::{IdentStr, Lowercase, TagName}; +use roc_module::ident::{Ident, IdentStr, Lowercase, TagName}; use roc_module::symbol::Symbol; +use roc_region::all::{Located, Region}; use roc_solve::solve; use roc_types::pretty_print::Parens; use roc_types::types::{Category, ErrorType, PatternCategory, Reason, RecordField, TypeExt}; @@ -10,34 +11,39 @@ use std::path::PathBuf; use crate::report::{Annotation, Report, RocDocAllocator, RocDocBuilder, Severity}; use ven_pretty::DocAllocator; +const DUPLICATE_NAME: &str = "DUPLICATE NAME"; const ADD_ANNOTATIONS: &str = r#"Can more type annotations be added? Type annotations always help me give more specific messages, and I think they could help a lot in this case"#; pub fn type_problem<'b>( alloc: &'b RocDocAllocator<'b>, filename: PathBuf, problem: solve::TypeError, -) -> Report<'b> { +) -> Option> { use solve::TypeError::*; - fn report(title: String, doc: RocDocBuilder<'_>, filename: PathBuf) -> Report<'_> { - Report { + fn report(title: String, doc: RocDocBuilder<'_>, filename: PathBuf) -> Option> { + Some(Report { title, filename, doc, severity: Severity::RuntimeError, - } + }) } match problem { - BadExpr(region, category, found, expected) => { - to_expr_report(alloc, filename, region, category, found, expected) - } - BadPattern(region, category, found, expected) => { - to_pattern_report(alloc, filename, region, category, found, expected) - } - CircularType(region, symbol, overall_type) => { - to_circular_report(alloc, filename, region, symbol, overall_type) - } + BadExpr(region, category, found, expected) => Some(to_expr_report( + alloc, filename, region, category, found, expected, + )), + BadPattern(region, category, found, expected) => Some(to_pattern_report( + alloc, filename, region, category, found, expected, + )), + CircularType(region, symbol, overall_type) => Some(to_circular_report( + alloc, + filename, + region, + symbol, + overall_type, + )), UnexposedLookup(symbol) => { let title = "UNRECOGNIZED NAME".to_string(); let doc = alloc @@ -97,12 +103,40 @@ pub fn type_problem<'b>( report(title, doc, filename) } + SolvedTypeError => None, // Don't re-report cascading errors - see https://github.com/rtfeldman/roc/pull/1711 + + Shadowed(original_region, shadow) => { + let doc = report_shadowing(alloc, original_region, shadow); + let title = DUPLICATE_NAME.to_string(); + + report(title, doc, filename) + } + other => panic!("unhandled bad type: {:?}", other), } } } } +fn report_shadowing<'b>( + alloc: &'b RocDocAllocator<'b>, + original_region: Region, + shadow: Located, +) -> RocDocBuilder<'b> { + let line = r#"Since these types have the same name, it's easy to use the wrong one on accident. Give one of them a new name."#; + + alloc.stack(vec![ + alloc + .text("The ") + .append(alloc.ident(shadow.value)) + .append(alloc.reflow(" name is first defined here:")), + alloc.region(original_region), + alloc.reflow("But then it's defined a second time here:"), + alloc.region(shadow.region), + alloc.reflow(line), + ]) +} + pub fn cyclic_alias<'b>( alloc: &'b RocDocAllocator<'b>, symbol: Symbol, diff --git a/compiler/reporting/src/report.rs b/compiler/reporting/src/report.rs index 28b615efe2..8a9b045f4c 100644 --- a/compiler/reporting/src/report.rs +++ b/compiler/reporting/src/report.rs @@ -128,6 +128,7 @@ impl<'b> Report<'b> { pub struct Palette<'a> { pub primary: &'a str, pub code_block: &'a str, + pub keyword: &'a str, pub variable: &'a str, pub type_variable: &'a str, pub structure: &'a str, @@ -146,6 +147,7 @@ pub struct Palette<'a> { pub const DEFAULT_PALETTE: Palette = Palette { primary: WHITE_CODE, code_block: WHITE_CODE, + keyword: GREEN_CODE, variable: BLUE_CODE, type_variable: YELLOW_CODE, structure: GREEN_CODE, @@ -810,6 +812,9 @@ where Symbol => { self.write_str(self.palette.variable)?; } + Keyword => { + self.write_str(self.palette.keyword)?; + } GutterBar => { self.write_str(self.palette.gutter_bar)?; } @@ -837,7 +842,7 @@ where ParserSuggestion => { self.write_str(self.palette.parser_suggestion)?; } - TypeBlock | GlobalTag | PrivateTag | RecordField | Keyword => { /* nothing yet */ } + TypeBlock | GlobalTag | PrivateTag | RecordField => { /* nothing yet */ } } self.style_stack.push(*annotation); Ok(()) @@ -851,11 +856,11 @@ where Some(annotation) => match annotation { Emphasized | Url | TypeVariable | Alias | Symbol | BinOp | Error | GutterBar | Typo | TypoSuggestion | ParserSuggestion | Structure | CodeBlock | PlainText - | LineNumber | Tip | Module | Header => { + | LineNumber | Tip | Module | Header | Keyword => { self.write_str(RESET_CODE)?; } - TypeBlock | GlobalTag | PrivateTag | RecordField | Keyword => { /* nothing yet */ } + TypeBlock | GlobalTag | PrivateTag | RecordField => { /* nothing yet */ } }, } Ok(()) diff --git a/compiler/reporting/tests/test_reporting.rs b/compiler/reporting/tests/test_reporting.rs index 7575aca5b0..14fb7279ad 100644 --- a/compiler/reporting/tests/test_reporting.rs +++ b/compiler/reporting/tests/test_reporting.rs @@ -154,8 +154,9 @@ mod test_reporting { } for problem in type_problems { - let report = type_problem(&alloc, filename.clone(), problem.clone()); - reports.push(report); + if let Some(report) = type_problem(&alloc, filename.clone(), problem.clone()) { + reports.push(report); + } } for problem in mono_problems { @@ -541,7 +542,7 @@ mod test_reporting { 8│ 4 -> bar baz "yay" ^^^ - these names seem close though: + Did you mean one of these? baz Nat @@ -739,7 +740,7 @@ mod test_reporting { 3│ theAdmin ^^^^^^^^ - these names seem close though: + Did you mean one of these? Decimal Dec @@ -1491,7 +1492,7 @@ mod test_reporting { 2│ { foo: 2 } -> foo ^^^ - these names seem close though: + Did you mean one of these? Bool U8 @@ -1947,7 +1948,7 @@ mod test_reporting { 2│ f = \_ -> ok 4 ^^ - these names seem close though: + Did you mean one of these? U8 f @@ -3634,8 +3635,8 @@ mod test_reporting { 1│ Foo.test ^^^^^^^^ - Is there an import missing? Perhaps there is a typo, these names seem - close: + Is there an import missing? Perhaps there is a typo. Did you mean one + of these? Bool Num @@ -5797,7 +5798,7 @@ mod test_reporting { 1│ [ "foo", bar("") ] ^^^ - these names seem close though: + Did you mean one of these? Nat Str diff --git a/compiler/test_gen/src/gen_list.rs b/compiler/test_gen/src/gen_list.rs index 1fd096f0a6..582b9acceb 100644 --- a/compiler/test_gen/src/gen_list.rs +++ b/compiler/test_gen/src/gen_list.rs @@ -2017,3 +2017,17 @@ fn lists_with_incompatible_type_param_in_if() { RocStr ); } + +#[test] +fn map_with_index_multi_record() { + // see https://github.com/rtfeldman/roc/issues/1700 + assert_evals_to!( + indoc!( + r#" + List.mapWithIndex [ { x: {}, y: {} } ] \_, _ -> {} + "# + ), + RocList::from_slice(&[((), ())]), + RocList<((), ())> + ); +} diff --git a/compiler/test_gen/src/gen_num.rs b/compiler/test_gen/src/gen_num.rs index 29bb479855..9d9d9e1390 100644 --- a/compiler/test_gen/src/gen_num.rs +++ b/compiler/test_gen/src/gen_num.rs @@ -1778,4 +1778,48 @@ mod gen_num { u32 ); } + + #[test] + fn when_on_i32() { + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + x : I32 + x = 0 + + main : I32 + main = + when x is + 0 -> 42 + _ -> -1 + "# + ), + 42, + i32 + ); + } + + #[test] + fn when_on_i16() { + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + x : I16 + x = 0 + + main : I16 + main = + when x is + 0 -> 42 + _ -> -1 + "# + ), + 42, + i16 + ); + } } diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs index 9a43cb8a78..d183b95d0b 100644 --- a/compiler/test_gen/src/gen_primitives.rs +++ b/compiler/test_gen/src/gen_primitives.rs @@ -2531,6 +2531,8 @@ fn pattern_match_unit_tag() { ); } +// see for why this is disabled on wasm32 https://github.com/rtfeldman/roc/issues/1687 +#[cfg(not(feature = "wasm-cli-run"))] #[test] fn mirror_llvm_alignment_padding() { // see https://github.com/rtfeldman/roc/issues/1569 @@ -2616,7 +2618,8 @@ fn lambda_set_struct_byte() { r = Red p1 = (\u -> r == u) - oneOfResult = List.map [p1, p1] (\p -> p Green) + foobarbaz = (\p -> p Green) + oneOfResult = List.map [p1, p1] foobarbaz when oneOfResult is _ -> 32 @@ -2779,3 +2782,127 @@ fn value_not_exposed_hits_panic() { i64 ); } + +#[test] +fn mix_function_and_closure() { + // see https://github.com/rtfeldman/roc/pull/1706 + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + # foo does not capture any variables + # but through unification will get a lambda set that does store information + # we must handle that correctly + foo = \x -> x + + bar = \y -> \_ -> y + + main : Str + main = + (if 1 == 1 then foo else (bar "nope nope nope")) "hello world" + "# + ), + RocStr::from_slice(b"hello world"), + RocStr + ); +} + +#[test] +fn mix_function_and_closure_level_of_indirection() { + // see https://github.com/rtfeldman/roc/pull/1706 + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + foo = \x -> x + + bar = \y -> \_ -> y + + f = (if 1 == 1 then foo else (bar "nope nope nope")) + + main : Str + main = + f "hello world" + "# + ), + RocStr::from_slice(b"hello world"), + RocStr + ); +} + +#[test] +fn do_pass_bool_byte_closure_layout() { + // see https://github.com/rtfeldman/roc/pull/1706 + // the distinction is actually important, dropping that info means some functions just get + // skipped + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + ## PARSER + + Parser a : List U8 -> List [Pair a (List U8)] + + + ## ANY + + # If succcessful, the any parser consumes one character + + any: Parser U8 + any = \inp -> + when List.first inp is + Ok u -> [Pair u (List.drop inp 1)] + _ -> [ ] + + + + ## SATISFY + + satisfy : (U8 -> Bool) -> Parser U8 + satisfy = \predicate -> + \input -> + walker = \(Pair u rest), accum -> + if predicate u then + Stop [ Pair u rest ] + + else + Stop accum + + List.walkUntil (any input) walker [] + + + + oneOf : List (Parser a) -> Parser a + oneOf = \parserList -> + \input -> + walker = \p, accum -> + output = p input + if List.len output == 1 then + Stop output + + else + Continue accum + + List.walkUntil parserList walker [] + + + satisfyA = satisfy (\u -> u == 97) # recognize 97 + satisfyB = satisfy (\u -> u == 98) # recognize 98 + + test1 = if List.len ((oneOf [satisfyA, satisfyB]) [97, 98, 99, 100] ) == 1 then "PASS" else "FAIL" + test2 = if List.len ((oneOf [satisfyA, satisfyB]) [98, 99, 100, 97] ) == 1 then "PASS" else "FAIL" + test3 = if List.len ((oneOf [satisfyB , satisfyA]) [98, 99, 100, 97] ) == 1 then "PASS" else "FAIL" + test4 = if List.len ((oneOf [satisfyA, satisfyB]) [99, 100, 101] ) == 0 then "PASS" else "FAIL" + + + main : Str + main = [test1, test2, test3, test4] |> Str.joinWith ", " + "# + ), + RocStr::from_slice(b"PASS, PASS, PASS, PASS"), + RocStr + ); +} diff --git a/compiler/test_gen/src/gen_records.rs b/compiler/test_gen/src/gen_records.rs index 18526a8107..9f6fbe478b 100644 --- a/compiler/test_gen/src/gen_records.rs +++ b/compiler/test_gen/src/gen_records.rs @@ -889,3 +889,26 @@ fn blue_and_absent() { i64 ); } + +#[test] +fn update_the_only_field() { + assert_evals_to!( + indoc!( + r#" + Model : { foo : I64 } + + model : Model + model = { foo: 3 } + + foo = 4 + + newModel : Model + newModel = { model & foo } + + newModel.foo + "# + ), + 4, + i64 + ); +} diff --git a/compiler/test_gen/src/helpers/eval.rs b/compiler/test_gen/src/helpers/eval.rs index dc4c377c84..be01cdc7ab 100644 --- a/compiler/test_gen/src/helpers/eval.rs +++ b/compiler/test_gen/src/helpers/eval.rs @@ -143,12 +143,13 @@ fn create_llvm_module<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/compiler/test_mono/generated/factorial.txt b/compiler/test_mono/generated/factorial.txt index c57881e00b..8abf602d63 100644 --- a/compiler/test_mono/generated/factorial.txt +++ b/compiler/test_mono/generated/factorial.txt @@ -6,7 +6,7 @@ procedure Num.26 (#Attr.2, #Attr.3): let Test.12 = lowlevel NumMul #Attr.2 #Attr.3; ret Test.12; -procedure Test.1 (Test.2, Test.3): +procedure Test.1 (Test.17, Test.18): joinpoint Test.7 Test.2 Test.3: let Test.15 = 0i64; let Test.16 = lowlevel Eq Test.15 Test.2; @@ -18,7 +18,7 @@ procedure Test.1 (Test.2, Test.3): let Test.11 = CallByName Num.26 Test.2 Test.3; jump Test.7 Test.10 Test.11; in - jump Test.7 Test.2 Test.3; + jump Test.7 Test.17 Test.18; procedure Test.0 (): let Test.5 = 10i64; diff --git a/compiler/test_mono/generated/has_none.txt b/compiler/test_mono/generated/has_none.txt index 5341f99624..b6e7dba474 100644 --- a/compiler/test_mono/generated/has_none.txt +++ b/compiler/test_mono/generated/has_none.txt @@ -1,4 +1,4 @@ -procedure Test.3 (Test.4): +procedure Test.3 (Test.29): joinpoint Test.13 Test.4: let Test.23 = 1i64; let Test.24 = GetTagId Test.4; @@ -18,7 +18,7 @@ procedure Test.3 (Test.4): let Test.7 = UnionAtIndex (Id 0) (Index 1) Test.4; jump Test.13 Test.7; in - jump Test.13 Test.4; + jump Test.13 Test.29; procedure Test.0 (): let Test.28 = 3i64; diff --git a/compiler/test_mono/generated/quicksort_help.txt b/compiler/test_mono/generated/quicksort_help.txt index 079b4fb480..79af94f700 100644 --- a/compiler/test_mono/generated/quicksort_help.txt +++ b/compiler/test_mono/generated/quicksort_help.txt @@ -10,7 +10,7 @@ procedure Num.27 (#Attr.2, #Attr.3): let Test.26 = lowlevel NumLt #Attr.2 #Attr.3; ret Test.26; -procedure Test.1 (Test.2, Test.3, Test.4): +procedure Test.1 (Test.27, Test.28, Test.29): joinpoint Test.12 Test.2 Test.3 Test.4: let Test.14 = CallByName Num.27 Test.3 Test.4; if Test.14 then @@ -29,7 +29,7 @@ procedure Test.1 (Test.2, Test.3, Test.4): else ret Test.2; in - jump Test.12 Test.2 Test.3 Test.4; + jump Test.12 Test.27 Test.28 Test.29; procedure Test.0 (): let Test.9 = Array []; diff --git a/compiler/test_mono/generated/somehow_drops_definitions.txt b/compiler/test_mono/generated/somehow_drops_definitions.txt index 15ddbd26c6..2e7a497919 100644 --- a/compiler/test_mono/generated/somehow_drops_definitions.txt +++ b/compiler/test_mono/generated/somehow_drops_definitions.txt @@ -1,43 +1,53 @@ procedure Num.24 (#Attr.2, #Attr.3): - let Test.24 = lowlevel NumAdd #Attr.2 #Attr.3; - ret Test.24; + let Test.27 = lowlevel NumAdd #Attr.2 #Attr.3; + ret Test.27; procedure Num.26 (#Attr.2, #Attr.3): - let Test.19 = lowlevel NumMul #Attr.2 #Attr.3; - ret Test.19; - -procedure Test.1 (): - let Test.25 = 1i64; - ret Test.25; - -procedure Test.2 (): - let Test.20 = 2i64; - ret Test.20; - -procedure Test.3 (Test.6): - let Test.23 = CallByName Test.1; - let Test.22 = CallByName Num.24 Test.6 Test.23; + let Test.22 = lowlevel NumMul #Attr.2 #Attr.3; ret Test.22; +procedure Test.1 (): + let Test.28 = 1i64; + ret Test.28; + +procedure Test.2 (): + let Test.23 = 2i64; + ret Test.23; + +procedure Test.3 (Test.6): + let Test.26 = CallByName Test.1; + let Test.25 = CallByName Num.24 Test.6 Test.26; + ret Test.25; + procedure Test.4 (Test.7): - let Test.18 = CallByName Test.2; - let Test.17 = CallByName Num.26 Test.7 Test.18; - ret Test.17; + let Test.21 = CallByName Test.2; + let Test.20 = CallByName Num.26 Test.7 Test.21; + ret Test.20; procedure Test.5 (Test.8, Test.9): - let Test.14 = CallByName Test.3 Test.9; - ret Test.14; + joinpoint Test.15 Test.14: + ret Test.14; + in + switch Test.8: + case 0: + let Test.16 = CallByName Test.3 Test.9; + jump Test.15 Test.16; + + default: + let Test.17 = CallByName Test.4 Test.9; + jump Test.15 Test.17; + procedure Test.0 (): - joinpoint Test.16 Test.12: + joinpoint Test.19 Test.12: let Test.13 = 42i64; let Test.11 = CallByName Test.5 Test.12 Test.13; ret Test.11; in - let Test.21 = true; - if Test.21 then - let Test.3 = Struct {}; - jump Test.16 Test.3; + let Test.24 = true; + if Test.24 then + let Test.3 = false; + jump Test.19 Test.3; else - let Test.4 = Struct {}; - jump Test.16 Test.4; + let Test.4 = true; + jump Test.19 Test.4; diff --git a/editor/src/editor/ed_error.rs b/editor/src/editor/ed_error.rs index b838fdef42..82b53f6bce 100644 --- a/editor/src/editor/ed_error.rs +++ b/editor/src/editor/ed_error.rs @@ -226,14 +226,6 @@ pub fn print_err(err: &EdError) { } } -/*pub fn print_ui_err(err: &UIError) { - eprintln!("{}", format!("{}", err).truecolor(255, 0, 0)); - - if let Some(backtrace) = ErrorCompat::backtrace(err) { - eprintln!("{}", color_backtrace(backtrace)); - } -}*/ - fn color_backtrace(backtrace: &snafu::Backtrace) -> String { let backtrace_str = format!("{}", backtrace); let backtrace_split = backtrace_str.split('\n'); diff --git a/examples/.gitignore b/examples/.gitignore index 874bbd86e5..c0e522cc76 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,3 +4,7 @@ app libhost.a roc_app.ll roc_app.bc +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file diff --git a/examples/benchmarks/platform/host.zig b/examples/benchmarks/platform/host.zig index 9f3e5efb5a..aa339ec306 100644 --- a/examples/benchmarks/platform/host.zig +++ b/examples/benchmarks/platform/host.zig @@ -23,16 +23,19 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed([*]u8) void; +extern fn roc__mainForHost_1_exposed_generic([*]u8) void; extern fn roc__mainForHost_size() i64; extern fn roc__mainForHost_1_Fx_caller(*const u8, [*]u8, [*]u8) void; extern fn roc__mainForHost_1_Fx_size() i64; extern fn roc__mainForHost_1_Fx_result_size() i64; -const Align = extern struct { a: usize, b: usize }; -extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; -extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; -extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +const Align = 2 * @alignOf(usize); +extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void; +extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -53,7 +56,7 @@ export fn roc_realloc(c_ptr: *c_void, new_size: usize, old_size: usize, alignmen stdout.print("realloc: {d} (alignment {d}, old_size {d})\n", .{ c_ptr, alignment, old_size }) catch unreachable; } - return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); + return realloc(@alignCast(Align, @ptrCast([*]u8, c_ptr)), new_size); } export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { @@ -62,7 +65,7 @@ export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { stdout.print("dealloc: {d} (alignment {d})\n", .{ c_ptr, alignment }) catch unreachable; } - free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); + free(@alignCast(Align, @ptrCast([*]u8, c_ptr))); } export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { @@ -74,37 +77,35 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; -pub fn main() u8 { +pub export fn main() callconv(.C) u8 { + const allocator = std.heap.page_allocator; + const size = @intCast(usize, roc__mainForHost_size()); - const raw_output = std.heap.c_allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; - roc__mainForHost_1_exposed(output); + roc__mainForHost_1_exposed_generic(output); - const flag = @ptrCast(*u64, @alignCast(@alignOf(u64), output)).*; + const closure_data_pointer = @ptrCast([*]u8, output); - if (flag == 0) { - // all is well - const closure_data_pointer = @ptrCast([*]u8, output[@sizeOf(u64)..size]); - - call_the_closure(closure_data_pointer); - } else { - const ptr = @ptrCast(*u32, output + @sizeOf(u64)); - const msg = @intToPtr([*:0]const u8, ptr.*); - const stderr = std.io.getStdErr().writer(); - stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; - - return 0; - } + call_the_closure(closure_data_pointer); var ts2: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; @@ -122,12 +123,14 @@ fn to_seconds(tms: std.os.timespec) f64 { } fn call_the_closure(closure_data_pointer: [*]u8) void { + const allocator = std.heap.page_allocator; + const size = roc__mainForHost_1_Fx_result_size(); - const raw_output = std.heap.c_allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } const flags: u8 = 0; diff --git a/examples/cli/platform/src/lib.rs b/examples/cli/platform/src/lib.rs index 2a98179f8e..2b24da5ff7 100644 --- a/examples/cli/platform/src/lib.rs +++ b/examples/cli/platform/src/lib.rs @@ -4,7 +4,7 @@ use core::alloc::Layout; use core::ffi::c_void; use core::mem::MaybeUninit; use libc; -use roc_std::{RocCallResult, RocStr}; +use roc_std::RocStr; use std::ffi::CStr; use std::os::raw::c_char; @@ -70,23 +70,11 @@ pub fn rust_main() -> isize { roc_main(buffer); - let output = buffer as *mut RocCallResult<()>; + let result = call_the_closure(buffer); - match (&*output).into() { - Ok(()) => { - let closure_data_ptr = buffer.offset(8); - let result = call_the_closure(closure_data_ptr as *const u8); + std::alloc::dealloc(buffer, layout); - std::alloc::dealloc(buffer, layout); - - result - } - Err(msg) => { - std::alloc::dealloc(buffer, layout); - - panic!("Roc failed with message: {}", msg); - } - } + result }; // Exit code @@ -105,15 +93,9 @@ unsafe fn call_the_closure(closure_data_ptr: *const u8) -> i64 { buffer as *mut u8, ); - let output = &*(buffer as *mut RocCallResult<()>); + std::alloc::dealloc(buffer, layout); - match output.into() { - Ok(_) => { - std::alloc::dealloc(buffer, layout); - 0 - } - Err(e) => panic!("failed with {}", e), - } + 0 } #[no_mangle] diff --git a/examples/effect/Main.roc b/examples/effect/Main.roc index 9e3dd5f506..15eb6ef6e8 100644 --- a/examples/effect/Main.roc +++ b/examples/effect/Main.roc @@ -5,4 +5,7 @@ app "effect-example" main : Effect.Effect {} main = - Effect.after Effect.getLine \lineThisThing -> Effect.putLine lineThisThing + Effect.after (Effect.getLine) \line -> + Effect.after (Effect.putLine "You entered: \(line)") \{} -> + Effect.after (Effect.putLine "It is known") \{} -> + Effect.always {} diff --git a/examples/effect/thing/platform-dir/host.zig b/examples/effect/thing/platform-dir/host.zig index 063d28973f..49fad7a204 100644 --- a/examples/effect/thing/platform-dir/host.zig +++ b/examples/effect/thing/platform-dir/host.zig @@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { return malloc(size); @@ -52,18 +54,29 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub export fn main() u8 { + const allocator = std.heap.page_allocator; + const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - const size = @intCast(usize, roc__mainForHost_size()); - const raw_output = std.heap.c_allocator.alloc(u8, size) catch unreachable; + // NOTE the return size can be zero, which will segfault. Always allocate at least 8 bytes + const size = std.math.max(8, @intCast(usize, roc__mainForHost_size())); + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } var ts1: std.os.timespec = undefined; @@ -71,21 +84,7 @@ pub export fn main() u8 { roc__mainForHost_1_exposed(output); - const elements = @ptrCast([*]u64, @alignCast(8, output)); - - var flag = elements[0]; - - if (flag == 0) { - // all is well - const closure_data_pointer = @ptrCast([*]u8, output[8..size]); - - call_the_closure(closure_data_pointer); - } else { - const msg = @intToPtr([*:0]const u8, elements[1]); - stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; - - return 0; - } + call_the_closure(output); var ts2: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; @@ -102,27 +101,20 @@ fn to_seconds(tms: std.os.timespec) f64 { } fn call_the_closure(closure_data_pointer: [*]u8) void { + const allocator = std.heap.page_allocator; + const size = roc__mainForHost_1_Fx_result_size(); - const raw_output = std.heap.c_allocator.alloc(u8, @intCast(usize, size)) catch unreachable; + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } const flags: u8 = 0; - roc__mainForHost_1_Fx_caller(&flags, closure_data_pointer, output); - const elements = @ptrCast([*]u64, @alignCast(8, output)); - - var flag = elements[0]; - - if (flag == 0) { - return; - } else { - unreachable; - } + return; } pub export fn roc_fx_getLine() str.RocStr { diff --git a/examples/fib/.gitignore b/examples/fib/.gitignore new file mode 100644 index 0000000000..41d1223f94 --- /dev/null +++ b/examples/fib/.gitignore @@ -0,0 +1,2 @@ +add +fib diff --git a/examples/fib/Fib.roc b/examples/fib/Fib.roc new file mode 100644 index 0000000000..e5098a023b --- /dev/null +++ b/examples/fib/Fib.roc @@ -0,0 +1,15 @@ +app "fib" + packages { base: "platform" } + imports [] + provides [ main ] to base + +main = \n -> fib n 0 1 + + +# the clever implementation requires join points +fib = \n, a, b -> + if n == 0 then + a + + else + fib (n - 1) b (a + b) diff --git a/examples/fib/platform/Package-Config.roc b/examples/fib/platform/Package-Config.roc new file mode 100644 index 0000000000..0990940738 --- /dev/null +++ b/examples/fib/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform examples/add + requires {}{ main : I64 -> I64 } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : I64 -> I64 +mainForHost = \a -> main a diff --git a/examples/fib/platform/host.zig b/examples/fib/platform/host.zig new file mode 100644 index 0000000000..111a84c27f --- /dev/null +++ b/examples/fib/platform/host.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +const mem = std.mem; +const Allocator = mem.Allocator; + +// NOTE the LLVM backend expects this signature +// extern fn roc__mainForHost_1_exposed(i64, *i64) void; +extern fn roc__mainForHost_1_exposed(i64) i64; + +const Align = extern struct { a: usize, b: usize }; +extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +const DEBUG: bool = false; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + if (DEBUG) { + var ptr = malloc(size); + const stdout = std.io.getStdOut().writer(); + stdout.print("alloc: {d} (alignment {d}, size {d})\n", .{ ptr, alignment, size }) catch unreachable; + return ptr; + } else { + return malloc(size); + } +} + +export fn roc_realloc(c_ptr: *c_void, new_size: usize, old_size: usize, alignment: u32) callconv(.C) ?*c_void { + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("realloc: {d} (alignment {d}, old_size {d})\n", .{ c_ptr, alignment, old_size }) catch unreachable; + } + + return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("dealloc: {d} (alignment {d})\n", .{ c_ptr, alignment }) catch unreachable; + } + + free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); +} + +export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { + _ = tag_id; + + const stderr = std.io.getStdErr().writer(); + const msg = @ptrCast([*:0]const u8, c_ptr); + stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; + std.process.exit(0); +} + +pub export fn main() u8 { + const stdout = std.io.getStdOut().writer(); + const stderr = std.io.getStdErr().writer(); + + // start time + var ts1: std.os.timespec = undefined; + std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; + + const result = roc__mainForHost_1_exposed(10); + + stdout.print("{d}\n", .{result}) catch unreachable; + + // end time + var ts2: std.os.timespec = undefined; + std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; + + const delta = to_seconds(ts2) - to_seconds(ts1); + + stderr.print("runtime: {d:.3}ms\n", .{delta * 1000}) catch unreachable; + + return 0; +} + +fn to_seconds(tms: std.os.timespec) f64 { + return @intToFloat(f64, tms.tv_sec) + (@intToFloat(f64, tms.tv_nsec) / 1_000_000_000.0); +} diff --git a/examples/hello-rust/platform/host.c b/examples/hello-rust/platform/host.c index 0378c69589..9b91965724 100644 --- a/examples/hello-rust/platform/host.c +++ b/examples/hello-rust/platform/host.c @@ -1,7 +1,12 @@ #include +#include extern int rust_main(); -int main() { - return rust_main(); +int main() { return rust_main(); } + +void *roc_memcpy(void *dest, const void *src, size_t n) { + return memcpy(dest, src, n); } + +void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); } \ No newline at end of file diff --git a/examples/hello-rust/platform/src/lib.rs b/examples/hello-rust/platform/src/lib.rs index 8726cab399..6a78b4db0c 100644 --- a/examples/hello-rust/platform/src/lib.rs +++ b/examples/hello-rust/platform/src/lib.rs @@ -3,12 +3,12 @@ use core::ffi::c_void; use core::mem::MaybeUninit; use libc::c_char; -use roc_std::{RocCallResult, RocStr}; +use roc_std::RocStr; use std::ffi::CStr; extern "C" { #[link_name = "roc__mainForHost_1_exposed"] - fn roc_main(output: *mut RocCallResult) -> (); + fn roc_main() -> RocStr; } #[no_mangle] @@ -46,25 +46,14 @@ pub unsafe fn roc_panic(c_ptr: *mut c_void, tag_id: u32) { #[no_mangle] pub fn rust_main() -> isize { - let mut call_result: MaybeUninit> = MaybeUninit::uninit(); - unsafe { - roc_main(call_result.as_mut_ptr()); + let roc_str = roc_main(); - let output = call_result.assume_init(); + let len = roc_str.len(); + let str_bytes = roc_str.get_bytes() as *const libc::c_void; - match output.into() { - Ok(roc_str) => { - let len = roc_str.len(); - let str_bytes = roc_str.get_bytes() as *const libc::c_void; - - if libc::write(1, str_bytes, len) < 0 { - panic!("Writing to stdout failed!"); - } - } - Err(msg) => { - panic!("Roc failed with message: {}", msg); - } + if libc::write(1, str_bytes, len) < 0 { + panic!("Writing to stdout failed!"); } } diff --git a/examples/hello-web/.gitignore b/examples/hello-web/.gitignore new file mode 100644 index 0000000000..227b499154 --- /dev/null +++ b/examples/hello-web/.gitignore @@ -0,0 +1,2 @@ +hello-web +*.wat diff --git a/examples/hello-web/Hello.roc b/examples/hello-web/Hello.roc new file mode 100644 index 0000000000..49f05ceacf --- /dev/null +++ b/examples/hello-web/Hello.roc @@ -0,0 +1,12 @@ +app "hello-web" + packages { base: "platform" } + imports [] + provides [ main ] to base + +greeting = + hi = "Hello" + name = "World" + + "\(hi), \(name)!" + +main = greeting diff --git a/examples/hello-web/README.md b/examples/hello-web/README.md new file mode 100644 index 0000000000..f0beccdb6b --- /dev/null +++ b/examples/hello-web/README.md @@ -0,0 +1,49 @@ +# Hello, World! + +To run, go to the project home directory and run: + +```bash +$ cargo run -- build --backend=wasm32 examples/hello-web/Hello.roc +``` + +Then `cd` into the example directory and run any web server that can handle WebAssembly. +For example with `http-server`: + +```bash +cd examples/hello-web +npm install -g http-server +http-server +``` + +Now open your browser at http://localhost:8080 + +## Design Notes + +This demonstrates the basic design of hosts: Roc code gets compiled into a pure +function (in this case, a thunk that always returns `"Hello, World!"`) and +then the host calls that function. Fundamentally, that's the whole idea! The host +might not even have a `main` - it could be a library, a plugin, anything. +Everything else is built on this basic "hosts calling linked pure functions" design. + +For example, things get more interesting when the compiled Roc function returns +a `Task` - that is, a tagged union data structure containing function pointers +to callback closures. This lets the Roc pure function describe arbitrary +chainable effects, which the host can interpret to perform I/O as requested by +the Roc program. (The tagged union `Task` would have a variant for each supported +I/O operation.) + +In this trivial example, it's very easy to line up the API between the host and +the Roc program. In a more involved host, this would be much trickier - especially +if the API were changing frequently during development. + +The idea there is to have a first-class concept of "glue code" which host authors +can write (it would be plain Roc code, but with some extra keywords that aren't +available in normal modules - kinda like `port module` in Elm), and which +describe both the Roc-host/C boundary as well as the Roc-host/Roc-app boundary. +Roc application authors only care about the Roc-host/Roc-app portion, and the +host author only cares about the Roc-host/C boundary when implementing the host. + +Using this glue code, the Roc compiler can generate C header files describing the +boundary. This not only gets us host compatibility with C compilers, but also +Rust FFI for free, because [`rust-bindgen`](https://github.com/rust-lang/rust-bindgen) +generates correct Rust FFI bindings from C headers. diff --git a/examples/hello-web/index.html b/examples/hello-web/index.html new file mode 100644 index 0000000000..87ca0302cb --- /dev/null +++ b/examples/hello-web/index.html @@ -0,0 +1,12 @@ + + +
+ + + + diff --git a/examples/hello-web/platform/Package-Config.roc b/examples/hello-web/platform/Package-Config.roc new file mode 100644 index 0000000000..377d5c0994 --- /dev/null +++ b/examples/hello-web/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform examples/hello-world + requires {}{ main : Str } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : Str +mainForHost = main diff --git a/examples/hello-web/platform/host.js b/examples/hello-web/platform/host.js new file mode 100644 index 0000000000..0d953eabff --- /dev/null +++ b/examples/hello-web/platform/host.js @@ -0,0 +1,51 @@ +async function roc_web_platform_run(wasm_filename, callback) { + const decoder = new TextDecoder(); + let memory_bytes; + let exit_code; + + function js_display_roc_string(str_bytes, str_len) { + const utf8_bytes = memory_bytes.subarray(str_bytes, str_bytes + str_len); + const js_string = decoder.decode(utf8_bytes); + callback(js_string); + } + + const importObj = { + wasi_snapshot_preview1: { + roc_panic: (_pointer, _tag_id) => { + throw 'Roc panicked!'; + } + }, + env: { + js_display_roc_string, + }, + }; + + let wasm; + + const response = await fetch(wasm_filename); + + if (WebAssembly.instantiateStreaming) { + // streaming API has better performance if available + wasm = await WebAssembly.instantiateStreaming(response, importObj); + } else { + const module_bytes = await response.arrayBuffer(); + wasm = await WebAssembly.instantiate(module_bytes, importObj); + } + + memory_bytes = new Uint8Array(wasm.instance.exports.memory.buffer); + + try { + wasm.instance.exports._start(); + } catch (e) { + const is_ok = e.message === "unreachable" && exit_code === 0; + if (!is_ok) { + console.error(e); + } + } +} + +if (typeof module !== 'undefined') { + module.exports = { + roc_web_platform_run, + }; +} diff --git a/examples/hello-web/platform/host.zig b/examples/hello-web/platform/host.zig new file mode 100644 index 0000000000..5d588d6912 --- /dev/null +++ b/examples/hello-web/platform/host.zig @@ -0,0 +1,72 @@ +const std = @import("std"); +const str = @import("str"); +const RocStr = str.RocStr; +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +const Align = extern struct { a: usize, b: usize }; +extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + _ = alignment; + + return malloc(size); +} + +export fn roc_realloc(c_ptr: *c_void, old_size: usize, new_size: usize, alignment: u32) callconv(.C) ?*c_void { + _ = old_size; + _ = alignment; + + return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + _ = alignment; + + free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); +} + +// NOTE roc_panic is provided in the JS file, so it can throw an exception + +const mem = std.mem; +const Allocator = mem.Allocator; + +extern fn roc__mainForHost_1_exposed(*RocCallResult) void; + +const RocCallResult = extern struct { flag: u64, content: RocStr }; + +const Unit = extern struct {}; + +extern fn js_display_roc_string(str_bytes: ?[*]u8, str_len: usize) void; + +pub fn main() u8 { + // make space for the result + var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; + + // actually call roc to populate the callresult + roc__mainForHost_1_exposed(&callresult); + + // display the result using JavaScript + js_display_roc_string(callresult.content.str_bytes, callresult.content.str_len); + + callresult.content.deinit(); + + return 0; +} diff --git a/examples/hello-web/test-node.js b/examples/hello-web/test-node.js new file mode 100644 index 0000000000..ce1eed5d6e --- /dev/null +++ b/examples/hello-web/test-node.js @@ -0,0 +1,25 @@ +/** + * Node.js test file for hello-web example + * We are not running this in CI currently, and Node.js is not a Roc dependency. + * But if you happen to have it, you can run this. + */ + +// Node doesn't have the fetch API +const fs = require("fs/promises"); +global.fetch = (filename) => + fs.readFile(filename).then((buffer) => ({ + arrayBuffer() { + return buffer; + }, + })); + +const { roc_web_platform_run } = require("./platform/host"); + +roc_web_platform_run("./hello-world.wasm", (string_from_roc) => { + const expected = "Hello, World!"; + if (string_from_roc !== expected) { + console.error(`Expected "${expected}", but got "${string_from_roc}"`); + process.exit(1); + } + console.log("OK"); +}); diff --git a/examples/hello-world/platform/host.c b/examples/hello-world/platform/host.c index f968d0c763..99ada123fe 100644 --- a/examples/hello-world/platform/host.c +++ b/examples/hello-world/platform/host.c @@ -1,89 +1,81 @@ -#include -#include -#include #include -#include #include +#include +#include +#include +#include -void* roc_alloc(size_t size, unsigned int alignment) { - return malloc(size); +void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); } + +void* roc_realloc(void* ptr, size_t old_size, size_t new_size, + unsigned int alignment) { + return realloc(ptr, new_size); } -void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) { - return realloc(ptr, new_size); -} - -void roc_dealloc(void* ptr, unsigned int alignment) { - free(ptr); -} +void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); } void roc_panic(void* ptr, unsigned int alignment) { - char* msg = (char *)ptr; - fprintf(stderr, "Application crashed with message\n\n %s\n\nShutting down\n", msg); - exit(0); + char* msg = (char*)ptr; + fprintf(stderr, + "Application crashed with message\n\n %s\n\nShutting down\n", msg); + exit(0); } +void* roc_memcpy(void* dest, const void* src, size_t n) { + return memcpy(dest, src, n); +} + +void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); } + struct RocStr { - char* bytes; - size_t len; + char* bytes; + size_t len; }; -bool is_small_str(struct RocStr str) { - return ((ssize_t)str.len) < 0; -} +bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; } // Determine the length of the string, taking into // account the small string optimization size_t roc_str_len(struct RocStr str) { - char* bytes = (char*)&str; - char last_byte = bytes[sizeof(str) - 1]; - char last_byte_xored = last_byte ^ 0b10000000; - size_t small_len = (size_t)(last_byte_xored); - size_t big_len = str.len; + char* bytes = (char*)&str; + char last_byte = bytes[sizeof(str) - 1]; + char last_byte_xored = last_byte ^ 0b10000000; + size_t small_len = (size_t)(last_byte_xored); + size_t big_len = str.len; - // Avoid branch misprediction costs by always - // determining both small_len and big_len, - // so this compiles to a cmov instruction. - if (is_small_str(str)) { - return small_len; - } else { - return big_len; - } + // Avoid branch misprediction costs by always + // determining both small_len and big_len, + // so this compiles to a cmov instruction. + if (is_small_str(str)) { + return small_len; + } else { + return big_len; + } } -struct RocCallResult { - size_t flag; - struct RocStr content; -}; - -extern void roc__mainForHost_1_exposed(struct RocCallResult *re); +extern struct RocStr roc__mainForHost_1_exposed(); int main() { - // Make space for the Roc call result - struct RocCallResult call_result; + struct RocStr str = roc__mainForHost_1_exposed(); - // Call Roc to populate call_result - roc__mainForHost_1_exposed(&call_result); + // Determine str_len and the str_bytes pointer, + // taking into account the small string optimization. + size_t str_len = roc_str_len(str); + char* str_bytes; - // Determine str_len and the str_bytes pointer, - // taking into account the small string optimization. - struct RocStr str = call_result.content; - size_t str_len = roc_str_len(str); - char* str_bytes; + if (is_small_str(str)) { + str_bytes = (char*)&str; + } else { + str_bytes = str.bytes; + } - if (is_small_str(str)) { - str_bytes = (char*)&str; - } else { - str_bytes = str.bytes; - } + // Write to stdout + if (write(1, str_bytes, str_len) >= 0) { + // Writing succeeded! + return 0; + } else { + printf("Error writing to stdout: %s\n", strerror(errno)); - // Write to stdout - if (write(1, str_bytes, str_len) >= 0) { - // Writing succeeded! - return 0; - } else { - printf("Error writing to stdout: %s\n", strerror(errno)); - - return 1; - } + return 1; + } } diff --git a/examples/hello-zig/platform/host.zig b/examples/hello-zig/platform/host.zig index 8384c996d3..3c10412eff 100644 --- a/examples/hello-zig/platform/host.zig +++ b/examples/hello-zig/platform/host.zig @@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { _ = alignment; @@ -51,12 +53,18 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; - -const RocCallResult = extern struct { flag: u64, content: RocStr }; +extern fn roc__mainForHost_1_exposed() RocStr; const Unit = extern struct {}; @@ -64,20 +72,17 @@ pub fn main() u8 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + var callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/examples/quicksort/platform/host.zig b/examples/quicksort/platform/host.zig index 1dd646d61d..fe8e90e490 100644 --- a/examples/quicksort/platform/host.zig +++ b/examples/quicksort/platform/host.zig @@ -20,41 +20,71 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(RocList, *RocCallResult) void; +extern fn roc__mainForHost_1_exposed(RocList) RocList; -extern fn malloc(size: usize) callconv(.C) ?*c_void; -extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; -extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; +const Align = extern struct { a: usize, b: usize }; +extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; + +const DEBUG: bool = false; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { - return malloc(size); + if (DEBUG) { + var ptr = malloc(size); + const stdout = std.io.getStdOut().writer(); + stdout.print("alloc: {d} (alignment {d}, size {d})\n", .{ ptr, alignment, size }) catch unreachable; + return ptr; + } else { + return malloc(size); + } } export fn roc_realloc(c_ptr: *c_void, new_size: usize, old_size: usize, alignment: u32) callconv(.C) ?*c_void { - return realloc(@alignCast(16, @ptrCast([*]u8, c_ptr)), new_size); + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("realloc: {d} (alignment {d}, old_size {d})\n", .{ c_ptr, alignment, old_size }) catch unreachable; + } + + return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); } export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { - free(@alignCast(16, @ptrCast([*]u8, c_ptr))); + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("dealloc: {d} (alignment {d})\n", .{ c_ptr, alignment }) catch unreachable; + } + + free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); } export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { + _ = tag_id; + const stderr = std.io.getStdErr().writer(); const msg = @ptrCast([*:0]const u8, c_ptr); stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + // warning! the array is currently stack-allocated so don't make this too big const NUM_NUMS = 100; const RocList = extern struct { elements: [*]i64, length: usize }; -const RocCallResult = extern struct { flag: usize, content: RocList }; - const Unit = extern struct {}; -pub export fn main() i32 { +pub export fn main() u8 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); @@ -65,25 +95,22 @@ pub export fn main() i32 { var numbers = raw_numbers[1..]; - for (numbers) |x, i| { + for (numbers) |_, i| { numbers[i] = @mod(@intCast(i64, i), 12); } const roc_list = RocList{ .elements = numbers, .length = NUM_NUMS }; - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = undefined }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(roc_list, &callresult); + var callresult = roc__mainForHost_1_exposed(roc_list); // stdout the result - const length = std.math.min(20, callresult.content.length); - var result = callresult.content.elements[0..length]; + const length = std.math.min(20, callresult.length); + var result = callresult.elements[0..length]; for (result) |x, i| { if (i == 0) { diff --git a/linker/Cargo.toml b/linker/Cargo.toml new file mode 100644 index 0000000000..843bf6caac --- /dev/null +++ b/linker/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "roc_linker" +version = "0.1.0" +authors = ["The Roc Contributors"] +license = "UPL-1.0" +repository = "https://github.com/rtfeldman/roc" +edition = "2018" +description = "A surgical linker for Roc" + +[lib] +name = "roc_linker" +path = "src/lib.rs" + +[[bin]] +name = "link" +path = "src/main.rs" +test = false +bench = false + +[dependencies] +roc_mono = { path = "../compiler/mono" } +roc_build = { path = "../compiler/build", default-features = false } +roc_collections = { path = "../compiler/collections" } +bumpalo = { version = "3.6", features = ["collections"] } +# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it +clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } +iced-x86 = "1.14" +memmap2 = "0.3" +object = { version = "0.26", features = ["read", "write"] } +serde = { version = "1.0", features = ["derive"] } +bincode = "1.3" +target-lexicon = "0.12.2" +tempfile = "3.1.0" diff --git a/linker/README.md b/linker/README.md new file mode 100644 index 0000000000..0d1e2d77ee --- /dev/null +++ b/linker/README.md @@ -0,0 +1,44 @@ +# The Roc Surgical Linker + +This linker has the goal of being extremely slim lined and fast. +It is focused on the scope of only linking platforms to Roc applications. +This restriction enables ignoring most of linking. + +## General Overview + +This linker is run in 2 phases: preprocessing and surigical linking. + +### Platform Preprocessor + +1. Dynamically link the platform to a dummy Roc application dynamic library +1. Create metadata related to Roc dynamically linked functions + - Symbols that need to be redefined + - Call locations that need to be modified for each symbol + - Locations of special roc functions (roc_alloc, roc_dealloc, builtins, etc) +1. Modify the main executable to no longer be dynamically link + - Delete dependency on dynamic library + - Remove symbols from the dynamic table (maybe add them to the regular table?) + - Delete GOT and PLT entries + - Remove relocations from the dynamic table + - Add extra header information about new text and data section at end of file + +### Surgical Linker + +1. Build off of preprocessed platform +1. Append text and data of application, dealing with app relocations +1. Surgically update all call locations in the platform +1. Surgically update call information in the application (also dealing with other relocations for builtins) + +## TODO (In a lightly prioritized order) + +- Run CLI tests and/or benchmarks with the Roc Linker. +- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig). +- Add Macho support + - Honestly should be almost exactly the same code. + This means we likely need to do a lot of refactoring to minimize the duplicate code. + The fun of almost but not quite the same. +- Add PE support + - As a prereq, we need roc building on Windows (I'm not sure it does currently). + - Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add. +- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl` +- Add a feature to the compiler to make this linker optional. diff --git a/linker/src/lib.rs b/linker/src/lib.rs new file mode 100644 index 0000000000..1a043fbce1 --- /dev/null +++ b/linker/src/lib.rs @@ -0,0 +1,1626 @@ +use bincode::{deserialize_from, serialize_into}; +use clap::{App, AppSettings, Arg, ArgMatches}; +use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind}; +use memmap2::{Mmap, MmapMut}; +use object::write; +use object::{elf, endian}; +use object::{ + Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian, + NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, + SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection, +}; +use roc_build::link::{rebuild_host, LinkType}; +use roc_collections::all::MutMap; +use roc_mono::ir::OptLevel; +use std::cmp::Ordering; +use std::convert::TryFrom; +use std::ffi::CStr; +use std::fs; +use std::io; +use std::io::{BufReader, BufWriter}; +use std::mem; +use std::os::raw::c_char; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; +use tempfile::Builder; + +mod metadata; + +pub const CMD_PREPROCESS: &str = "preprocess"; +pub const CMD_SURGERY: &str = "surgery"; +pub const FLAG_VERBOSE: &str = "verbose"; +pub const FLAG_TIME: &str = "time"; + +pub const EXEC: &str = "EXEC"; +pub const METADATA: &str = "METADATA"; +pub const SHARED_LIB: &str = "SHARED_LIB"; +pub const APP: &str = "APP"; +pub const OUT: &str = "OUT"; + +const MIN_SECTION_ALIGNMENT: usize = 0x40; + +// TODO: Analyze if this offset is always correct. +const PLT_ADDRESS_OFFSET: u64 = 0x10; + +fn report_timing(label: &str, duration: Duration) { + println!("\t{:9.3} ms {}", duration.as_secs_f64() * 1000.0, label,); +} + +pub fn build_app<'a>() -> App<'a> { + App::new("link") + .about("Preprocesses a platform and surgically links it to an application.") + .setting(AppSettings::SubcommandRequiredElseHelp) + .subcommand( + App::new(CMD_PREPROCESS) + .about("Preprocesses a dynamically linked platform to prepare for linking.") + .arg( + Arg::with_name(EXEC) + .help("The dynamically linked platform executable") + .required(true), + ) + .arg( + Arg::with_name(METADATA) + .help("Where to save the metadata from preprocessing") + .required(true), + ) + .arg( + Arg::with_name(OUT) + .help("The modified version of the dynamically linked platform executable") + .required(true), + ) + .arg( + Arg::with_name(SHARED_LIB) + .help("The name of the shared library used in building the platform") + .default_value("libapp.so"), + ) + .arg( + Arg::with_name(FLAG_VERBOSE) + .long(FLAG_VERBOSE) + .short('v') + .help("Enable verbose printing") + .required(false), + ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .short('t') + .help("Print timing information") + .required(false), + ), + ) + .subcommand( + App::new(CMD_SURGERY) + .about("Links a preprocessed platform with a Roc application.") + .arg( + Arg::with_name(APP) + .help("The Roc application object file waiting to be linked") + .required(true), + ) + .arg( + Arg::with_name(METADATA) + .help("The metadata created by preprocessing the platform") + .required(true), + ) + .arg( + Arg::with_name(OUT) + .help( + "The modified version of the dynamically linked platform. \ + It will be consumed to make linking faster.", + ) + .required(true), + ) + .arg( + Arg::with_name(FLAG_VERBOSE) + .long(FLAG_VERBOSE) + .short('v') + .help("Enable verbose printing") + .required(false), + ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .short('t') + .help("Print timing information") + .required(false), + ), + ) +} + +pub fn supported(link_type: &LinkType, target: &Triple) -> bool { + link_type == &LinkType::Executable + && target.architecture == target_lexicon::Architecture::X86_64 + && target.operating_system == target_lexicon::OperatingSystem::Linux + && target.binary_format == target_lexicon::BinaryFormat::Elf +} + +pub fn build_and_preprocess_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + exposed_to_host: Vec, +) -> io::Result<()> { + let dummy_lib = host_input_path.with_file_name("libapp.so"); + generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?; + rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib)); + let dynhost = host_input_path.with_file_name("dynhost"); + let metadata = host_input_path.with_file_name("metadata"); + let prehost = host_input_path.with_file_name("preprocessedhost"); + if preprocess_impl( + dynhost.to_str().unwrap(), + metadata.to_str().unwrap(), + prehost.to_str().unwrap(), + dummy_lib.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to preprocess host"); + } + Ok(()) +} + +pub fn link_preprocessed_host( + _target: &Triple, + host_input_path: &Path, + roc_app_obj: &Path, + binary_path: &Path, +) -> io::Result<()> { + let metadata = host_input_path.with_file_name("metadata"); + if surgery_impl( + roc_app_obj.to_str().unwrap(), + metadata.to_str().unwrap(), + binary_path.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to surgically link host"); + } + Ok(()) +} + +fn generate_dynamic_lib( + _target: &Triple, + exposed_to_host: Vec, + dummy_lib_path: &Path, +) -> io::Result<()> { + let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?; + let dummy_obj_file = dummy_obj_file.path(); + + // TODO deal with other architectures here. + let mut out_object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let text_section = out_object.section_id(write::StandardSection::Text); + for sym in exposed_to_host { + // TODO properly generate this list. + for name in &[ + format!("roc__{}_1_exposed", sym), + format!("roc__{}_1_Fx_caller", sym), + format!("roc__{}_1_Fx_size", sym), + format!("roc__{}_1_Fx_result_size", sym), + format!("roc__{}_size", sym), + ] { + out_object.add_symbol(write::Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } + } + std::fs::write( + &dummy_obj_file, + out_object.write().expect("failed to build output object"), + ) + .expect("failed to write object to file"); + + let output = Command::new("ld") + .args(&[ + "-shared", + "-soname", + dummy_lib_path.file_name().unwrap().to_str().unwrap(), + dummy_obj_file.to_str().unwrap(), + "-o", + dummy_lib_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + + if !output.status.success() { + match std::str::from_utf8(&output.stderr) { + Ok(stderr) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was:\n{}", + stderr + ), + Err(utf8_err) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})", + utf8_err + ), + } + } + Ok(()) +} + +pub fn preprocess(matches: &ArgMatches) -> io::Result { + preprocess_impl( + matches.value_of(EXEC).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.value_of(SHARED_LIB).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} +// TODO: Most of this file is a mess of giant functions just to check if things work. +// Clean it all up and refactor nicely. +fn preprocess_impl( + exec_filename: &str, + metadata_filename: &str, + out_filename: &str, + shared_lib_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { + let total_start = SystemTime::now(); + let exec_parsing_start = total_start; + let exec_file = fs::File::open(exec_filename)?; + let exec_mmap = unsafe { Mmap::map(&exec_file)? }; + let exec_data = &*exec_mmap; + let exec_obj = match object::File::parse(exec_data) { + Ok(obj) => obj, + Err(err) => { + println!("Failed to parse executable file: {}", err); + return Ok(-1); + } + }; + let exec_header = load_struct_inplace::>(exec_data, 0); + + let ph_offset = exec_header.e_phoff.get(NativeEndian); + let ph_ent_size = exec_header.e_phentsize.get(NativeEndian); + let ph_num = exec_header.e_phnum.get(NativeEndian); + let sh_offset = exec_header.e_shoff.get(NativeEndian); + let sh_ent_size = exec_header.e_shentsize.get(NativeEndian); + let sh_num = exec_header.e_shnum.get(NativeEndian); + if verbose { + println!(); + println!("PH Offset: {:+x}", ph_offset); + println!("PH Entry Size: {}", ph_ent_size); + println!("PH Entry Count: {}", ph_num); + println!("SH Offset: {:+x}", sh_offset); + println!("SH Entry Size: {}", sh_ent_size); + println!("SH Entry Count: {}", sh_num); + } + + // TODO: Deal with other file formats and architectures. + let format = exec_obj.format(); + if format != BinaryFormat::Elf { + println!("File Format, {:?}, not supported", format); + return Ok(-1); + } + let arch = exec_obj.architecture(); + if arch != Architecture::X86_64 { + println!("Architecture, {:?}, not supported", arch); + return Ok(-1); + } + + let mut md: metadata::Metadata = Default::default(); + + for sym in exec_obj.symbols().filter(|sym| { + sym.is_definition() && sym.name().is_ok() && sym.name().unwrap().starts_with("roc_") + }) { + let name = sym.name().unwrap().to_string(); + // special exceptions for memcpy and memset. + if &name == "roc_memcpy" { + md.roc_symbol_vaddresses + .insert("memcpy".to_string(), sym.address() as u64); + } else if name == "roc_memset" { + md.roc_symbol_vaddresses + .insert("memset".to_string(), sym.address() as u64); + } + md.roc_symbol_vaddresses.insert(name, sym.address() as u64); + } + + if verbose { + println!( + "Found roc symbol definitions: {:+x?}", + md.roc_symbol_vaddresses + ); + } + + let exec_parsing_duration = exec_parsing_start.elapsed().unwrap(); + + // Extract PLT related information for app functions. + let symbol_and_plt_processing_start = SystemTime::now(); + let (plt_address, plt_offset) = match exec_obj.section_by_name(".plt") { + Some(section) => { + let file_offset = match section.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset, + _ => { + println!("Surgical linking does not work with compressed plt section"); + return Ok(-1); + } + }; + (section.address(), file_offset) + } + None => { + println!("Failed to find PLT section. Probably an malformed executable."); + return Ok(-1); + } + }; + if verbose { + println!("PLT Address: {:+x}", plt_address); + println!("PLT File Offset: {:+x}", plt_offset); + } + + // TODO: it looks like we may need to support global data host relocations. + // Rust host look to be using them by default instead of the plt. + // I think this is due to first linking into a static lib and then linking to the c wrapper. + let plt_relocs = (match exec_obj.dynamic_relocations() { + Some(relocs) => relocs, + None => { + println!("Executable never calls any application functions."); + println!("No work to do. Probably an invalid input."); + return Ok(-1); + } + }) + .map(|(_, reloc)| reloc) + .filter(|reloc| reloc.kind() == RelocationKind::Elf(7)); + + let app_syms: Vec = exec_obj + .dynamic_symbols() + .filter(|sym| { + sym.is_undefined() && sym.name().is_ok() && sym.name().unwrap().starts_with("roc_") + }) + .collect(); + for sym in app_syms.iter() { + let name = sym.name().unwrap().to_string(); + md.app_functions.push(name.clone()); + md.surgeries.insert(name.clone(), vec![]); + md.dynamic_symbol_indices.insert(name, sym.index().0 as u64); + } + if verbose { + println!(); + println!("PLT Symbols for App Functions"); + for symbol in app_syms.iter() { + println!("{}: {:+x?}", symbol.index().0, symbol); + } + } + + let mut app_func_addresses: MutMap = MutMap::default(); + for (i, reloc) in plt_relocs.enumerate() { + for symbol in app_syms.iter() { + if reloc.target() == RelocationTarget::Symbol(symbol.index()) { + let func_address = (i as u64 + 1) * PLT_ADDRESS_OFFSET + plt_address; + let func_offset = (i as u64 + 1) * PLT_ADDRESS_OFFSET + plt_offset; + app_func_addresses.insert(func_address, symbol.name().unwrap()); + md.plt_addresses.insert( + symbol.name().unwrap().to_string(), + (func_offset, func_address), + ); + break; + } + } + } + + if verbose { + println!(); + println!("App Function Address Map: {:+x?}", app_func_addresses); + } + let symbol_and_plt_processing_duration = symbol_and_plt_processing_start.elapsed().unwrap(); + + let text_disassembly_start = SystemTime::now(); + let text_sections: Vec
= exec_obj + .sections() + .filter(|sec| { + let name = sec.name(); + name.is_ok() && name.unwrap().starts_with(".text") + }) + .collect(); + if text_sections.is_empty() { + println!("No text sections found. This application has no code."); + return Ok(-1); + } + if verbose { + println!(); + println!("Text Sections"); + for sec in text_sections.iter() { + println!("{:+x?}", sec); + } + } + + if verbose { + println!(); + println!("Analyzing instuctions for branches"); + } + let mut indirect_warning_given = false; + for sec in text_sections { + let (file_offset, compressed) = match sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => (range.offset, false), + Ok(range) => (range.offset, true), + Err(err) => { + println!( + "Issues dealing with section compression for {:+x?}: {}", + sec, err + ); + return Ok(-1); + } + }; + + let data = match sec.uncompressed_data() { + Ok(data) => data, + Err(err) => { + println!("Failed to load text section, {:+x?}: {}", sec, err); + return Ok(-1); + } + }; + let mut decoder = Decoder::with_ip(64, &data, sec.address(), DecoderOptions::NONE); + let mut inst = Instruction::default(); + + while decoder.can_decode() { + decoder.decode_out(&mut inst); + + // Note: This gets really complex fast if we want to support more than basic calls/jumps. + // A lot of them have to load addresses into registers/memory so we would have to discover that value. + // Would probably require some static code analysis and would be impossible in some cases. + // As an alternative we can leave in the calls to the plt, but change the plt to jmp to the static function. + // That way any indirect call will just have the overhead of an extra jump. + match inst.try_op_kind(0) { + // Relative Offsets. + Ok(OpKind::NearBranch16 | OpKind::NearBranch32 | OpKind::NearBranch64) => { + let target = inst.near_branch_target(); + if let Some(func_name) = app_func_addresses.get(&target) { + if compressed { + println!("Surgical linking does not work with compressed text sections: {:+x?}", sec); + return Ok(-1); + } + + if verbose { + println!( + "Found branch from {:+x} to {:+x}({})", + inst.ip(), + target, + func_name + ); + } + + // TODO: Double check these offsets are always correct. + // We may need to do a custom offset based on opcode instead. + let op_kind = inst.op_code().try_op_kind(0).unwrap(); + let op_size: u8 = match op_kind { + OpCodeOperandKind::br16_1 | OpCodeOperandKind::br32_1 => 1, + OpCodeOperandKind::br16_2 => 2, + OpCodeOperandKind::br32_4 | OpCodeOperandKind::br64_4 => 4, + _ => { + println!( + "Ran into an unknown operand kind when analyzing branches: {:?}", + op_kind + ); + return Ok(-1); + } + }; + let offset = inst.next_ip() - op_size as u64 - sec.address() + file_offset; + if verbose { + println!( + "\tNeed to surgically replace {} bytes at file offset {:+x}", + op_size, offset, + ); + println!( + "\tIts current value is {:+x?}", + &exec_data[offset as usize..(offset + op_size as u64) as usize] + ) + } + md.surgeries + .get_mut(*func_name) + .unwrap() + .push(metadata::SurgeryEntry { + file_offset: offset, + virtual_offset: inst.next_ip(), + size: op_size, + }); + } + } + Ok(OpKind::FarBranch16 | OpKind::FarBranch32) => { + println!( + "Found branch type instruction that is not yet support: {:+x?}", + inst + ); + return Ok(-1); + } + Ok(_) => { + if (inst.is_call_far_indirect() + || inst.is_call_near_indirect() + || inst.is_jmp_far_indirect() + || inst.is_jmp_near_indirect()) + && !indirect_warning_given + && verbose + { + indirect_warning_given = true; + println!(); + println!("Cannot analyaze through indirect jmp type instructions"); + println!("Most likely this is not a problem, but it could mean a loss in optimizations"); + println!(); + } + } + Err(err) => { + println!("Failed to decode assembly: {}", err); + return Ok(-1); + } + } + } + } + let text_disassembly_duration = text_disassembly_start.elapsed().unwrap(); + + let scanning_dynamic_deps_start = SystemTime::now(); + + let dyn_sec = match exec_obj.section_by_name(".dynamic") { + Some(sec) => sec, + None => { + println!("There must be a dynamic section in the executable"); + return Ok(-1); + } + }; + let dyn_offset = match dyn_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed dynamic section"); + return Ok(-1); + } + }; + md.dynamic_section_offset = dyn_offset as u64; + + let dynstr_sec = match exec_obj.section_by_name(".dynstr") { + Some(sec) => sec, + None => { + println!("There must be a dynstr section in the executable"); + return Ok(-1); + } + }; + let dynstr_data = match dynstr_sec.uncompressed_data() { + Ok(data) => data, + Err(err) => { + println!("Failed to load dynstr section: {}", err); + return Ok(-1); + } + }; + + let shared_lib_name = Path::new(shared_lib_filename) + .file_name() + .unwrap() + .to_str() + .unwrap(); + + let mut dyn_lib_index = 0; + let mut shared_lib_index = None; + loop { + let dyn_tag = u64::from_le_bytes( + <[u8; 8]>::try_from( + &exec_data[dyn_offset + dyn_lib_index * 16..dyn_offset + dyn_lib_index * 16 + 8], + ) + .unwrap(), + ); + if dyn_tag == 0 { + break; + } else if dyn_tag == 1 { + let dynstr_off = u64::from_le_bytes( + <[u8; 8]>::try_from( + &exec_data + [dyn_offset + dyn_lib_index * 16 + 8..dyn_offset + dyn_lib_index * 16 + 16], + ) + .unwrap(), + ) as usize; + let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8; + let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap(); + if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name { + shared_lib_index = Some(dyn_lib_index); + if verbose { + println!( + "Found shared lib in dynamic table at index: {}", + dyn_lib_index + ); + } + } + } + + dyn_lib_index += 1; + } + let dynamic_lib_count = dyn_lib_index as usize; + + if shared_lib_index.is_none() { + println!("Shared lib not found as a dependency of the executable"); + return Ok(-1); + } + let shared_lib_index = shared_lib_index.unwrap(); + + let scanning_dynamic_deps_duration = scanning_dynamic_deps_start.elapsed().unwrap(); + + let symtab_sec = match exec_obj.section_by_name(".symtab") { + Some(sec) => sec, + None => { + println!("There must be a symtab section in the executable"); + return Ok(-1); + } + }; + let symtab_offset = match symtab_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed symtab section"); + return Ok(-1); + } + }; + md.symbol_table_section_offset = symtab_offset as u64; + md.symbol_table_size = symtab_sec.size(); + + let dynsym_sec = match exec_obj.section_by_name(".dynsym") { + Some(sec) => sec, + None => { + println!("There must be a dynsym section in the executable"); + return Ok(-1); + } + }; + let dynsym_offset = match dynsym_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed dynsym section"); + return Ok(-1); + } + }; + md.dynamic_symbol_table_section_offset = dynsym_offset as u64; + + let mut got_sections: Vec<(usize, usize)> = vec![]; + for sec in exec_obj + .sections() + .filter(|sec| sec.name().is_ok() && sec.name().unwrap().starts_with(".got")) + { + match sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => got_sections.push((range.offset as usize, range.uncompressed_size as usize)), + _ => { + println!("Surgical linking does not work with compressed got sections"); + return Ok(-1); + } + } + } + + let platform_gen_start = SystemTime::now(); + + // Copy header and shift everything to enable more program sections. + let added_header_count = 2; + md.added_byte_count = ph_ent_size as u64 * added_header_count; + md.added_byte_count = md.added_byte_count + + (MIN_SECTION_ALIGNMENT as u64 - md.added_byte_count % MIN_SECTION_ALIGNMENT as u64); + let ph_end = ph_offset as usize + ph_num as usize * ph_ent_size as usize; + let physical_shift_start = ph_end as u64; + + md.exec_len = exec_data.len() as u64 + md.added_byte_count; + let out_file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(out_filename)?; + out_file.set_len(md.exec_len)?; + let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; + + out_mmap[..ph_end].copy_from_slice(&exec_data[..ph_end]); + + let program_headers = load_structs_inplace_mut::>( + &mut out_mmap, + ph_offset as usize, + ph_num as usize, + ); + let mut first_load_found = false; + let mut virtual_shift_start = 0; + for ph in program_headers.iter() { + let p_type = ph.p_type.get(NativeEndian); + if p_type == elf::PT_LOAD && ph.p_offset.get(NativeEndian) == 0 { + first_load_found = true; + md.load_align_constraint = ph.p_align.get(NativeEndian); + virtual_shift_start = physical_shift_start + ph.p_vaddr.get(NativeEndian); + } + } + if !first_load_found { + println!("Executable does not load any data at 0x00000000"); + println!("Probably input the wrong file as the executable"); + return Ok(-1); + } + if verbose { + println!( + "Shifting all data after: {:+x}({:+x})", + physical_shift_start, virtual_shift_start + ); + } + + // Shift all of the program headers. + for ph in program_headers.iter_mut() { + let p_type = ph.p_type.get(NativeEndian); + let p_offset = ph.p_offset.get(NativeEndian); + if (p_type == elf::PT_LOAD && p_offset == 0) || p_type == elf::PT_PHDR { + // Extend length for the first segment and the program header. + ph.p_filesz = endian::U64::new( + LittleEndian, + ph.p_filesz.get(NativeEndian) + md.added_byte_count, + ); + ph.p_memsz = endian::U64::new( + LittleEndian, + ph.p_memsz.get(NativeEndian) + md.added_byte_count, + ); + } else { + // Shift if needed. + if physical_shift_start <= p_offset { + ph.p_offset = endian::U64::new(LittleEndian, p_offset + md.added_byte_count); + } + let p_vaddr = ph.p_vaddr.get(NativeEndian); + if virtual_shift_start <= p_vaddr { + ph.p_vaddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count); + ph.p_paddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count); + } + } + } + + // Get last segment virtual address. + let last_segment_vaddr = program_headers + .iter() + .filter(|ph| ph.p_type.get(NativeEndian) != elf::PT_GNU_STACK) + .map(|ph| ph.p_vaddr.get(NativeEndian) + ph.p_memsz.get(NativeEndian)) + .max() + .unwrap(); + + // Copy the rest of the file shifted as needed. + out_mmap[physical_shift_start as usize + md.added_byte_count as usize..] + .copy_from_slice(&exec_data[physical_shift_start as usize..]); + + // Update all sections for shift for extra program headers. + let section_headers = load_structs_inplace_mut::>( + &mut out_mmap, + sh_offset as usize + md.added_byte_count as usize, + sh_num as usize, + ); + + let mut rel_sections: Vec<(u64, u64)> = vec![]; + let mut rela_sections: Vec<(u64, u64)> = vec![]; + for sh in section_headers.iter_mut() { + let sh_offset = sh.sh_offset.get(NativeEndian); + let sh_addr = sh.sh_addr.get(NativeEndian); + if physical_shift_start <= sh_offset { + sh.sh_offset = endian::U64::new(LittleEndian, sh_offset + md.added_byte_count); + } + if virtual_shift_start <= sh_addr { + sh.sh_addr = endian::U64::new(LittleEndian, sh_addr + md.added_byte_count); + } + + // Record every relocation section. + let sh_type = sh.sh_type.get(NativeEndian); + if sh_type == elf::SHT_REL { + rel_sections.push((sh_offset, sh.sh_size.get(NativeEndian))); + } else if sh_type == elf::SHT_RELA { + rela_sections.push((sh_offset, sh.sh_size.get(NativeEndian))); + } + } + + // Get last section virtual address. + let last_section_vaddr = section_headers + .iter() + .map(|sh| sh.sh_addr.get(NativeEndian) + sh.sh_size.get(NativeEndian)) + .max() + .unwrap(); + + // Calculate end virtual address for new segment. + // TODO: potentially remove md.load_align_constraint here. I think we should be able to cram things together. + md.last_vaddr = + std::cmp::max(last_section_vaddr, last_segment_vaddr) + md.load_align_constraint; + + // Update all relocations for shift for extra program headers. + for (sec_offset, sec_size) in rel_sections { + let relocations = load_structs_inplace_mut::>( + &mut out_mmap, + sec_offset as usize + md.added_byte_count as usize, + sec_size as usize / mem::size_of::>(), + ); + for rel in relocations.iter_mut() { + let r_offset = rel.r_offset.get(NativeEndian); + if virtual_shift_start <= r_offset { + rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count); + } + } + } + for (sec_offset, sec_size) in rela_sections { + let relocations = load_structs_inplace_mut::>( + &mut out_mmap, + sec_offset as usize + md.added_byte_count as usize, + sec_size as usize / mem::size_of::>(), + ); + for rel in relocations.iter_mut() { + let r_offset = rel.r_offset.get(NativeEndian); + if virtual_shift_start <= r_offset { + rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count); + // Deal with potential adjusts to absolute jumps. + // TODO: Verify other relocation types. + if rel.r_type(LittleEndian, false) == elf::R_X86_64_RELATIVE { + let r_addend = rel.r_addend.get(LittleEndian); + rel.r_addend + .set(LittleEndian, r_addend + md.added_byte_count as i64); + } + } + } + } + + // Update dynamic table entries for shift for extra program headers. + let dyn_offset = md.dynamic_section_offset + md.added_byte_count; + + let dyns = load_structs_inplace_mut::>( + &mut out_mmap, + dyn_offset as usize, + dynamic_lib_count, + ); + for mut d in dyns { + match d.d_tag.get(NativeEndian) as u32 { + // I believe this is the list of symbols that need to be update if addresses change. + // I am less sure about the symbols from GNU_HASH down. + elf::DT_INIT + | elf::DT_FINI + | elf::DT_PLTGOT + | elf::DT_HASH + | elf::DT_STRTAB + | elf::DT_SYMTAB + | elf::DT_RELA + | elf::DT_REL + | elf::DT_DEBUG + | elf::DT_JMPREL + | elf::DT_INIT_ARRAY + | elf::DT_FINI_ARRAY + | elf::DT_PREINIT_ARRAY + | elf::DT_SYMTAB_SHNDX + | elf::DT_GNU_HASH + | elf::DT_TLSDESC_PLT + | elf::DT_TLSDESC_GOT + | elf::DT_GNU_CONFLICT + | elf::DT_GNU_LIBLIST + | elf::DT_CONFIG + | elf::DT_DEPAUDIT + | elf::DT_AUDIT + | elf::DT_PLTPAD + | elf::DT_MOVETAB + | elf::DT_SYMINFO + | elf::DT_VERSYM + | elf::DT_VERDEF + | elf::DT_VERNEED => { + let d_addr = d.d_val.get(NativeEndian); + if virtual_shift_start <= d_addr { + d.d_val = endian::U64::new(LittleEndian, d_addr + md.added_byte_count); + } + } + _ => {} + } + } + + // Update symbol table entries for shift for extra program headers. + let symtab_offset = md.symbol_table_section_offset + md.added_byte_count; + let symtab_size = md.symbol_table_size as usize; + + let symbols = load_structs_inplace_mut::>( + &mut out_mmap, + symtab_offset as usize, + symtab_size / mem::size_of::>(), + ); + + for sym in symbols { + let addr = sym.st_value.get(NativeEndian); + if virtual_shift_start <= addr { + sym.st_value = endian::U64::new(LittleEndian, addr + md.added_byte_count); + } + } + + // Update all data in the global offset table. + for (offset, size) in got_sections { + let global_offsets = load_structs_inplace_mut::>( + &mut out_mmap, + offset as usize + md.added_byte_count as usize, + size / mem::size_of::>(), + ); + for go in global_offsets.iter_mut() { + let go_addr = go.get(NativeEndian); + if physical_shift_start <= go_addr { + go.set(LittleEndian, go_addr + md.added_byte_count); + } + } + } + + // TODO: look into shifting all of the debug info and eh_frames. + + // Delete shared library from the dynamic table. + let out_ptr = out_mmap.as_mut_ptr(); + unsafe { + std::ptr::copy( + out_ptr.add(dyn_offset as usize + 16 * (shared_lib_index + 1)), + out_ptr.add(dyn_offset as usize + 16 * shared_lib_index), + 16 * (dynamic_lib_count - shared_lib_index), + ); + } + + // Update main elf header for extra data. + let mut file_header = + load_struct_inplace_mut::>(&mut out_mmap, 0); + file_header.e_shoff = endian::U64::new( + LittleEndian, + file_header.e_shoff.get(NativeEndian) + md.added_byte_count, + ); + let e_entry = file_header.e_entry.get(NativeEndian); + if virtual_shift_start <= e_entry { + file_header.e_entry = endian::U64::new(LittleEndian, e_entry + md.added_byte_count); + } + file_header.e_phnum = endian::U16::new(LittleEndian, ph_num + added_header_count as u16); + + let platform_gen_duration = platform_gen_start.elapsed().unwrap(); + + if verbose { + println!(); + println!("{:+x?}", md); + } + + let saving_metadata_start = SystemTime::now(); + // This block ensure that the metadata is fully written and timed before continuing. + { + let output = fs::File::create(metadata_filename)?; + let output = BufWriter::new(output); + if let Err(err) = serialize_into(output, &md) { + println!("Failed to serialize metadata: {}", err); + return Ok(-1); + }; + } + let saving_metadata_duration = saving_metadata_start.elapsed().unwrap(); + + let flushing_data_start = SystemTime::now(); + out_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(out_mmap); + drop(out_file); + let flushing_data_duration = flushing_data_start.elapsed().unwrap(); + + let total_duration = total_start.elapsed().unwrap(); + + if verbose || time { + println!(); + println!("Timings"); + report_timing("Executable Parsing", exec_parsing_duration); + report_timing( + "Symbol and PLT Processing", + symbol_and_plt_processing_duration, + ); + report_timing("Text Disassembly", text_disassembly_duration); + report_timing("Scanning Dynamic Deps", scanning_dynamic_deps_duration); + report_timing("Generate Modified Platform", platform_gen_duration); + report_timing("Saving Metadata", saving_metadata_duration); + report_timing("Flushing Data to Disk", flushing_data_duration); + report_timing( + "Other", + total_duration + - exec_parsing_duration + - symbol_and_plt_processing_duration + - text_disassembly_duration + - scanning_dynamic_deps_duration + - platform_gen_duration + - saving_metadata_duration + - flushing_data_duration, + ); + report_timing("Total", total_duration); + } + + Ok(0) +} + +pub fn surgery(matches: &ArgMatches) -> io::Result { + surgery_impl( + matches.value_of(APP).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} + +fn surgery_impl( + app_filename: &str, + metadata_filename: &str, + out_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { + let total_start = SystemTime::now(); + let loading_metadata_start = total_start; + let input = fs::File::open(metadata_filename)?; + let input = BufReader::new(input); + let md: metadata::Metadata = match deserialize_from(input) { + Ok(data) => data, + Err(err) => { + println!("Failed to deserialize metadata: {}", err); + return Ok(-1); + } + }; + let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); + + let app_parsing_start = SystemTime::now(); + let app_file = fs::File::open(app_filename)?; + let app_mmap = unsafe { Mmap::map(&app_file)? }; + let app_data = &*app_mmap; + let app_obj = match object::File::parse(app_data) { + Ok(obj) => obj, + Err(err) => { + println!("Failed to parse application file: {}", err); + return Ok(-1); + } + }; + let app_parsing_duration = app_parsing_start.elapsed().unwrap(); + + let exec_parsing_start = SystemTime::now(); + let exec_file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(out_filename)?; + + let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; + exec_file.set_len(max_out_len)?; + + let mut exec_mmap = unsafe { MmapMut::map_mut(&exec_file)? }; + let elf64 = exec_mmap[4] == 2; + let litte_endian = exec_mmap[5] == 1; + if !elf64 || !litte_endian { + println!("Only 64bit little endian elf currently supported for surgery"); + return Ok(-1); + } + let exec_header = load_struct_inplace::>(&exec_mmap, 0); + + let ph_offset = exec_header.e_phoff.get(NativeEndian); + let ph_ent_size = exec_header.e_phentsize.get(NativeEndian); + let ph_num = exec_header.e_phnum.get(NativeEndian); + let sh_offset = exec_header.e_shoff.get(NativeEndian); + let sh_ent_size = exec_header.e_shentsize.get(NativeEndian); + let sh_num = exec_header.e_shnum.get(NativeEndian); + if verbose { + println!(); + println!("Is Elf64: {}", elf64); + println!("Is Little Endian: {}", litte_endian); + println!("PH Offset: {:+x}", ph_offset); + println!("PH Entry Size: {}", ph_ent_size); + println!("PH Entry Count: {}", ph_num); + println!("SH Offset: {:+x}", sh_offset); + println!("SH Entry Size: {}", sh_ent_size); + println!("SH Entry Count: {}", sh_num); + } + let exec_parsing_duration = exec_parsing_start.elapsed().unwrap(); + + let out_gen_start = SystemTime::now(); + // Backup section header table. + let sh_size = sh_ent_size as usize * sh_num as usize; + let mut sh_tab = vec![]; + sh_tab.extend_from_slice(&exec_mmap[sh_offset as usize..sh_offset as usize + sh_size]); + + let mut offset = sh_offset as usize; + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + + let new_rodata_section_offset = offset; + + // Align physical and virtual address of new segment. + let mut virt_offset = align_to_offset_by_constraint( + md.last_vaddr as usize, + offset, + md.load_align_constraint as usize, + ); + let new_rodata_section_vaddr = virt_offset; + if verbose { + println!(); + println!( + "New Virtual Rodata Section Address: {:+x?}", + new_rodata_section_vaddr + ); + } + + // First decide on sections locations and then recode every exact symbol locations. + + // Copy sections and resolve their symbols/relocations. + let symbols = app_obj.symbols().collect::>(); + let mut section_offset_map: MutMap = MutMap::default(); + let mut symbol_vaddr_map: MutMap = MutMap::default(); + let mut app_func_vaddr_map: MutMap = MutMap::default(); + let mut app_func_size_map: MutMap = MutMap::default(); + + // TODO: Does Roc ever create a data section? I think no cause it would mess up fully functional guarantees. + // If not we never need to think about it, but we should double check. + let rodata_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".rodata")) + .collect(); + + // bss section is like rodata section, but it has zero file size and non-zero virtual size. + let bss_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".bss")) + .collect(); + + let text_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".text")) + .collect(); + if text_sections.is_empty() { + println!("No text sections found. This application has no code."); + return Ok(-1); + } + + // Calculate addresses and load symbols. + // Note, it is important the bss sections come after the rodata sections. + for sec in rodata_sections + .iter() + .chain(bss_sections.iter()) + .chain(text_sections.iter()) + { + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + virt_offset = + align_to_offset_by_constraint(virt_offset, offset, md.load_align_constraint as usize); + if verbose { + println!( + "Section, {}, is being put at offset: {:+x}(virt: {:+x})", + sec.name().unwrap(), + offset, + virt_offset + ) + } + section_offset_map.insert(sec.index(), (offset, virt_offset)); + for sym in symbols.iter() { + if sym.section() == SymbolSection::Section(sec.index()) { + let name = sym.name().unwrap_or_default().to_string(); + if !md.roc_symbol_vaddresses.contains_key(&name) { + symbol_vaddr_map.insert(sym.index(), virt_offset + sym.address() as usize); + } + if md.app_functions.contains(&name) { + app_func_vaddr_map.insert(name.clone(), virt_offset + sym.address() as usize); + app_func_size_map.insert(name, sym.size()); + } + } + } + let section_size = match sec.file_range() { + Some((_, size)) => size, + None => 0, + }; + if sec.name().unwrap_or_default().starts_with(".bss") { + // bss sections only modify the virtual size. + virt_offset += sec.size() as usize; + } else if section_size != sec.size() { + println!( + "We do not deal with non bss sections that have different on disk and in memory sizes" + ); + return Ok(-1); + } else { + offset += section_size as usize; + virt_offset += sec.size() as usize; + } + } + if verbose { + println!("Data Relocation Offsets: {:+x?}", symbol_vaddr_map); + println!("Found App Function Symbols: {:+x?}", app_func_vaddr_map); + } + + let (new_text_section_offset, new_text_section_vaddr) = text_sections + .iter() + .map(|sec| section_offset_map.get(&sec.index()).unwrap()) + .min() + .unwrap(); + let (new_text_section_offset, new_text_section_vaddr) = + (*new_text_section_offset, *new_text_section_vaddr); + + // Move data and deal with relocations. + for sec in rodata_sections + .iter() + .chain(bss_sections.iter()) + .chain(text_sections.iter()) + { + let data = match sec.data() { + Ok(data) => data, + Err(err) => { + println!( + "Failed to load data for section, {:+x?}: {}", + sec.name().unwrap(), + err + ); + return Ok(-1); + } + }; + let (section_offset, section_virtual_offset) = + section_offset_map.get(&sec.index()).unwrap(); + let (section_offset, section_virtual_offset) = (*section_offset, *section_virtual_offset); + exec_mmap[section_offset..section_offset + data.len()].copy_from_slice(data); + // Deal with definitions and relocations for this section. + if verbose { + println!(); + println!( + "Processing Relocations for Section: {:+x?} @ {:+x} (virt: {:+x})", + sec, section_offset, section_virtual_offset + ); + } + for rel in sec.relocations() { + if verbose { + println!("\tFound Relocation: {:+x?}", rel); + } + match rel.1.target() { + RelocationTarget::Symbol(index) => { + let target_offset = if let Some(target_offset) = symbol_vaddr_map.get(&index) { + if verbose { + println!( + "\t\tRelocation targets symbol in app at: {:+x}", + target_offset + ); + } + Some(*target_offset as i64) + } else { + app_obj + .symbol_by_index(index) + .and_then(|sym| sym.name()) + .ok() + .and_then(|name| { + md.roc_symbol_vaddresses.get(name).map(|address| { + let vaddr = (*address + md.added_byte_count) as i64; + if verbose { + println!( + "\t\tRelocation targets symbol in host: {} @ {:+x}", + name, vaddr + ); + } + vaddr + }) + }) + }; + + if let Some(target_offset) = target_offset { + let virt_base = section_virtual_offset as usize + rel.0 as usize; + let base = section_offset as usize + rel.0 as usize; + let target: i64 = match rel.1.kind() { + RelocationKind::Relative | RelocationKind::PltRelative => { + target_offset - virt_base as i64 + rel.1.addend() + } + x => { + println!("Relocation Kind not yet support: {:?}", x); + return Ok(-1); + } + }; + if verbose { + println!( + "\t\tRelocation base location: {:+x} (virt: {:+x})", + base, virt_base + ); + println!("\t\tFinal relocation target offset: {:+x}", target); + } + match rel.1.size() { + 32 => { + let data = (target as i32).to_le_bytes(); + exec_mmap[base..base + 4].copy_from_slice(&data); + } + 64 => { + let data = target.to_le_bytes(); + exec_mmap[base..base + 8].copy_from_slice(&data); + } + x => { + println!("Relocation size not yet supported: {}", x); + return Ok(-1); + } + } + } else if matches!(app_obj.symbol_by_index(index), Ok(sym) if ["__divti3", "__udivti3"].contains(&sym.name().unwrap_or_default())) + { + // Explicitly ignore some symbols that are currently always linked. + continue; + } else { + println!( + "Undefined Symbol in relocation, {:+x?}: {:+x?}", + rel, + app_obj.symbol_by_index(index) + ); + return Ok(-1); + } + } + + _ => { + println!("Relocation target not yet support: {:+x?}", rel); + return Ok(-1); + } + } + } + } + + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + let new_sh_offset = offset; + exec_mmap[offset..offset + sh_size].copy_from_slice(&sh_tab); + offset += sh_size; + + // Flush app only data to speed up write to disk. + exec_mmap.flush_async_range( + new_rodata_section_offset, + offset - new_rodata_section_offset, + )?; + + // TODO: look into merging symbol tables, debug info, and eh frames to enable better debugger experience. + + // Add 2 new sections and segments. + let new_section_count = 2; + offset += new_section_count * sh_ent_size as usize; + let section_headers = load_structs_inplace_mut::>( + &mut exec_mmap, + new_sh_offset as usize, + sh_num as usize + new_section_count, + ); + + let new_rodata_section_size = new_text_section_offset as u64 - new_rodata_section_offset as u64; + let new_rodata_section_virtual_size = + new_text_section_vaddr as u64 - new_rodata_section_vaddr as u64; + let new_text_section_vaddr = new_rodata_section_vaddr as u64 + new_rodata_section_size as u64; + let new_text_section_size = new_sh_offset as u64 - new_text_section_offset as u64; + + let new_rodata_section = &mut section_headers[section_headers.len() - 2]; + new_rodata_section.sh_name = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_type = endian::U32::new(LittleEndian, elf::SHT_PROGBITS); + new_rodata_section.sh_flags = endian::U64::new(LittleEndian, (elf::SHF_ALLOC) as u64); + new_rodata_section.sh_addr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_section.sh_offset = endian::U64::new(LittleEndian, new_rodata_section_offset as u64); + new_rodata_section.sh_size = endian::U64::new(LittleEndian, new_rodata_section_size); + new_rodata_section.sh_link = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_info = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_addralign = endian::U64::new(LittleEndian, 16); + new_rodata_section.sh_entsize = endian::U64::new(LittleEndian, 0); + + let new_text_section_index = section_headers.len() - 1; + let new_text_section = &mut section_headers[new_text_section_index]; + new_text_section.sh_name = endian::U32::new(LittleEndian, 0); + new_text_section.sh_type = endian::U32::new(LittleEndian, elf::SHT_PROGBITS); + new_text_section.sh_flags = + endian::U64::new(LittleEndian, (elf::SHF_ALLOC | elf::SHF_EXECINSTR) as u64); + new_text_section.sh_addr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_section.sh_offset = endian::U64::new(LittleEndian, new_text_section_offset as u64); + new_text_section.sh_size = endian::U64::new(LittleEndian, new_text_section_size); + new_text_section.sh_link = endian::U32::new(LittleEndian, 0); + new_text_section.sh_info = endian::U32::new(LittleEndian, 0); + new_text_section.sh_addralign = endian::U64::new(LittleEndian, 16); + new_text_section.sh_entsize = endian::U64::new(LittleEndian, 0); + + // Reload and update file header and size. + let file_header = load_struct_inplace_mut::>(&mut exec_mmap, 0); + file_header.e_shoff = endian::U64::new(LittleEndian, new_sh_offset as u64); + file_header.e_shnum = endian::U16::new(LittleEndian, sh_num + new_section_count as u16); + + // Add 2 new segments that match the new sections. + let program_headers = load_structs_inplace_mut::>( + &mut exec_mmap, + ph_offset as usize, + ph_num as usize, + ); + let new_rodata_segment = &mut program_headers[program_headers.len() - 2]; + new_rodata_segment.p_type = endian::U32::new(LittleEndian, elf::PT_LOAD); + new_rodata_segment.p_flags = endian::U32::new(LittleEndian, elf::PF_R); + new_rodata_segment.p_offset = endian::U64::new(LittleEndian, new_rodata_section_offset as u64); + new_rodata_segment.p_vaddr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_segment.p_paddr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_segment.p_filesz = endian::U64::new(LittleEndian, new_rodata_section_size); + new_rodata_segment.p_memsz = endian::U64::new(LittleEndian, new_rodata_section_virtual_size); + new_rodata_segment.p_align = endian::U64::new(LittleEndian, md.load_align_constraint); + + let new_text_segment = &mut program_headers[program_headers.len() - 1]; + new_text_segment.p_type = endian::U32::new(LittleEndian, elf::PT_LOAD); + new_text_segment.p_flags = endian::U32::new(LittleEndian, elf::PF_R | elf::PF_X); + new_text_segment.p_offset = endian::U64::new(LittleEndian, new_text_section_offset as u64); + new_text_segment.p_vaddr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_segment.p_paddr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_segment.p_filesz = endian::U64::new(LittleEndian, new_text_section_size); + new_text_segment.p_memsz = endian::U64::new(LittleEndian, new_text_section_size); + new_text_segment.p_align = endian::U64::new(LittleEndian, md.load_align_constraint); + + // Update calls from platform and dynamic symbols. + let dynsym_offset = md.dynamic_symbol_table_section_offset + md.added_byte_count; + + for func_name in md.app_functions { + let virt_offset = match app_func_vaddr_map.get(&func_name) { + Some(offset) => *offset as u64, + None => { + println!("Function, {}, was not defined by the app", &func_name); + return Ok(-1); + } + }; + if verbose { + println!( + "Updating calls to {} to the address: {:+x}", + &func_name, virt_offset + ); + } + + for s in md.surgeries.get(&func_name).unwrap_or(&vec![]) { + if verbose { + println!("\tPerforming surgery: {:+x?}", s); + } + match s.size { + 4 => { + let target = (virt_offset as i64 + - (s.virtual_offset + md.added_byte_count) as i64) + as i32; + if verbose { + println!("\tTarget Jump: {:+x}", target); + } + let data = target.to_le_bytes(); + exec_mmap[(s.file_offset + md.added_byte_count) as usize + ..(s.file_offset + md.added_byte_count) as usize + 4] + .copy_from_slice(&data); + } + x => { + println!("Surgery size not yet supported: {}", x); + return Ok(-1); + } + } + } + + // Replace plt call code with just a jump. + // This is a backup incase we missed a call to the plt. + if let Some((plt_off, plt_vaddr)) = md.plt_addresses.get(&func_name) { + let plt_off = (*plt_off + md.added_byte_count) as usize; + let plt_vaddr = *plt_vaddr + md.added_byte_count; + let jmp_inst_len = 5; + let target = (virt_offset as i64 - (plt_vaddr as i64 + jmp_inst_len as i64)) as i32; + if verbose { + println!("\tPLT: {:+x}, {:+x}", plt_off, plt_vaddr); + println!("\tTarget Jump: {:+x}", target); + } + let data = target.to_le_bytes(); + exec_mmap[plt_off] = 0xE9; + exec_mmap[plt_off + 1..plt_off + jmp_inst_len].copy_from_slice(&data); + for i in jmp_inst_len..PLT_ADDRESS_OFFSET as usize { + exec_mmap[plt_off + i] = 0x90; + } + } + + if let Some(i) = md.dynamic_symbol_indices.get(&func_name) { + let sym = load_struct_inplace_mut::>( + &mut exec_mmap, + dynsym_offset as usize + *i as usize * mem::size_of::>(), + ); + sym.st_shndx = endian::U16::new(LittleEndian, new_text_section_index as u16); + sym.st_value = endian::U64::new(LittleEndian, virt_offset as u64); + sym.st_size = endian::U64::new( + LittleEndian, + match app_func_size_map.get(&func_name) { + Some(size) => *size, + None => { + println!("Size missing for: {}", &func_name); + return Ok(-1); + } + }, + ); + } + } + + let out_gen_duration = out_gen_start.elapsed().unwrap(); + + let flushing_data_start = SystemTime::now(); + exec_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(exec_mmap); + exec_file.set_len(offset as u64 + 1)?; + drop(exec_file); + let flushing_data_duration = flushing_data_start.elapsed().unwrap(); + + // Make sure the final executable has permision to execute. + let mut perms = fs::metadata(out_filename)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(out_filename, perms)?; + + let total_duration = total_start.elapsed().unwrap(); + + if verbose || time { + println!(); + println!("Timings"); + report_timing("Loading Metadata", loading_metadata_duration); + report_timing("Executable Parsing", exec_parsing_duration); + report_timing("Application Parsing", app_parsing_duration); + report_timing("Output Generation", out_gen_duration); + report_timing("Flushing Data to Disk", flushing_data_duration); + report_timing( + "Other", + total_duration + - loading_metadata_duration + - exec_parsing_duration + - app_parsing_duration + - out_gen_duration + - flushing_data_duration, + ); + report_timing("Total", total_duration); + } + Ok(0) +} + +fn align_by_constraint(offset: usize, constraint: usize) -> usize { + if offset % constraint == 0 { + offset + } else { + offset + constraint - (offset % constraint) + } +} + +fn align_to_offset_by_constraint( + current_offset: usize, + target_offset: usize, + constraint: usize, +) -> usize { + let target_remainder = target_offset % constraint; + let current_remainder = current_offset % constraint; + match target_remainder.cmp(¤t_remainder) { + Ordering::Greater => current_offset + (target_remainder - current_remainder), + Ordering::Less => current_offset + ((target_remainder + constraint) - current_remainder), + Ordering::Equal => current_offset, + } +} + +fn load_struct_inplace(bytes: &[u8], offset: usize) -> &T { + &load_structs_inplace(bytes, offset, 1)[0] +} + +fn load_struct_inplace_mut(bytes: &mut [u8], offset: usize) -> &mut T { + &mut load_structs_inplace_mut(bytes, offset, 1)[0] +} + +fn load_structs_inplace(bytes: &[u8], offset: usize, count: usize) -> &[T] { + let (head, body, tail) = + unsafe { bytes[offset..offset + count * mem::size_of::()].align_to::() }; + assert!(head.is_empty(), "Data was not aligned"); + assert_eq!(count, body.len(), "Failed to load all structs"); + assert!(tail.is_empty(), "End of data was not aligned"); + body +} + +fn load_structs_inplace_mut(bytes: &mut [u8], offset: usize, count: usize) -> &mut [T] { + let (head, body, tail) = + unsafe { bytes[offset..offset + count * mem::size_of::()].align_to_mut::() }; + assert!(head.is_empty(), "Data was not aligned"); + assert_eq!(count, body.len(), "Failed to load all structs"); + assert!(tail.is_empty(), "End of data was not aligned"); + body +} diff --git a/linker/src/main.rs b/linker/src/main.rs new file mode 100644 index 0000000000..6c1f341f9d --- /dev/null +++ b/linker/src/main.rs @@ -0,0 +1,20 @@ +use roc_linker::{build_app, preprocess, surgery, CMD_PREPROCESS, CMD_SURGERY}; +use std::io; + +fn main() -> io::Result<()> { + let matches = build_app().get_matches(); + + let exit_code = match matches.subcommand_name() { + None => Ok::(-1), + Some(CMD_PREPROCESS) => { + let sub_matches = matches.subcommand_matches(CMD_PREPROCESS).unwrap(); + preprocess(sub_matches) + } + Some(CMD_SURGERY) => { + let sub_matches = matches.subcommand_matches(CMD_SURGERY).unwrap(); + surgery(sub_matches) + } + _ => unreachable!(), + }?; + std::process::exit(exit_code); +} diff --git a/linker/src/metadata.rs b/linker/src/metadata.rs new file mode 100644 index 0000000000..16f06232cd --- /dev/null +++ b/linker/src/metadata.rs @@ -0,0 +1,31 @@ +use roc_collections::all::MutMap; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +pub struct SurgeryEntry { + pub file_offset: u64, + pub virtual_offset: u64, + pub size: u8, +} + +// TODO: Reanalyze each piece of data in this struct. +// I think a number of them can be combined to reduce string duplication. +// Also I think a few of them aren't need. +// For example, I think preprocessing can deal with all shifting and remove the need for added_byte_count. +#[derive(Default, Serialize, Deserialize, PartialEq, Debug)] +pub struct Metadata { + pub app_functions: Vec, + // offset followed by address. + pub plt_addresses: MutMap, + pub surgeries: MutMap>, + pub dynamic_symbol_indices: MutMap, + pub roc_symbol_vaddresses: MutMap, + pub exec_len: u64, + pub load_align_constraint: u64, + pub added_byte_count: u64, + pub last_vaddr: u64, + pub dynamic_section_offset: u64, + pub dynamic_symbol_table_section_offset: u64, + pub symbol_table_section_offset: u64, + pub symbol_table_size: u64, +} diff --git a/linker/tests/fib/.gitignore b/linker/tests/fib/.gitignore new file mode 100644 index 0000000000..a3c0d77f6d --- /dev/null +++ b/linker/tests/fib/.gitignore @@ -0,0 +1,11 @@ +fib + +zig-cache +zig-out + +*.o + +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file diff --git a/linker/tests/fib/Main.roc b/linker/tests/fib/Main.roc new file mode 100644 index 0000000000..646fdbea75 --- /dev/null +++ b/linker/tests/fib/Main.roc @@ -0,0 +1,15 @@ +app "fib" + packages { base: "platform" } + imports [] + provides [ main ] to base + +main : U64 -> U64 +main = \index -> + fibHelp index 0 1 + +fibHelp : U64, U64, U64 -> U64 +fibHelp = \index, parent, grandparent -> + if index == 0 then + parent + else + fibHelp (index - 1) grandparent (parent + grandparent) \ No newline at end of file diff --git a/linker/tests/fib/README.md b/linker/tests/fib/README.md new file mode 100644 index 0000000000..0f1af10077 --- /dev/null +++ b/linker/tests/fib/README.md @@ -0,0 +1,48 @@ +# Hello, World! + +To run, `cd` into this directory and run: + +```bash +$ cargo run Hello.roc +``` + +To run in release mode instead, do: + +```bash +$ cargo run --release Hello.roc +``` + +## Troubleshooting + +If you encounter `cannot find -lc++`, run the following for ubuntu `sudo apt install libc++-dev`. + +## Design Notes + +This demonstrates the basic design of hosts: Roc code gets compiled into a pure +function (in this case, a thunk that always returns `"Hello, World!"`) and +then the host calls that function. Fundamentally, that's the whole idea! The host +might not even have a `main` - it could be a library, a plugin, anything. +Everything else is built on this basic "hosts calling linked pure functions" design. + +For example, things get more interesting when the compiled Roc function returns +a `Task` - that is, a tagged union data structure containing function pointers +to callback closures. This lets the Roc pure function describe arbitrary +chainable effects, which the host can interpret to perform I/O as requested by +the Roc program. (The tagged union `Task` would have a variant for each supported +I/O operation.) + +In this trivial example, it's very easy to line up the API between the host and +the Roc program. In a more involved host, this would be much trickier - especially +if the API were changing frequently during development. + +The idea there is to have a first-class concept of "glue code" which host authors +can write (it would be plain Roc code, but with some extra keywords that aren't +available in normal modules - kinda like `port module` in Elm), and which +describe both the Roc-host/C boundary as well as the Roc-host/Roc-app boundary. +Roc application authors only care about the Roc-host/Roc-app portion, and the +host author only cares about the Roc-host/C boundary when implementing the host. + +Using this glue code, the Roc compiler can generate C header files describing the +boundary. This not only gets us host compatibility with C compilers, but also +Rust FFI for free, because [`rust-bindgen`](https://github.com/rust-lang/rust-bindgen) +generates correct Rust FFI bindings from C headers. diff --git a/linker/tests/fib/platform/Package-Config.roc b/linker/tests/fib/platform/Package-Config.roc new file mode 100644 index 0000000000..d93cd7c258 --- /dev/null +++ b/linker/tests/fib/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform tests/fib + requires {}{ main : U64 -> U64 } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : U64 -> U64 +mainForHost = \arg -> main arg # workaround for https://github.com/rtfeldman/roc/issues/1622 \ No newline at end of file diff --git a/linker/tests/fib/platform/app.zig b/linker/tests/fib/platform/app.zig new file mode 100644 index 0000000000..105908633f --- /dev/null +++ b/linker/tests/fib/platform/app.zig @@ -0,0 +1 @@ +export fn roc__mainForHost_1_exposed(_i: i64, _result: *u64) void {} diff --git a/linker/tests/fib/platform/build.zig b/linker/tests/fib/platform/build.zig new file mode 100644 index 0000000000..deb36d6c78 --- /dev/null +++ b/linker/tests/fib/platform/build.zig @@ -0,0 +1,33 @@ +const Builder = @import("std").build.Builder; + +pub fn build(b: *Builder) void { + // Standard target options allows the person running `zig build` to choose + // what target to build for. Here we do not override the defaults, which + // means any target is allowed, and the default is native. Other options + // for restricting supported target set are available. + const target = b.standardTargetOptions(.{}); + + // Standard release options allow the person running `zig build` to select + // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. + const mode = b.standardReleaseOptions(); + + const app = b.addSharedLibrary("app", "app.zig", .unversioned); + app.setTarget(target); + app.setBuildMode(mode); + app.install(); + + const exe = b.addExecutable("dynhost", "host.zig"); + exe.pie = true; + exe.strip = true; + exe.setTarget(target); + exe.setBuildMode(mode); + exe.linkLibrary(app); + exe.linkLibC(); + exe.install(); + + const run_cmd = exe.run(); + run_cmd.step.dependOn(b.getInstallStep()); + + const run_step = b.step("run", "Run the app"); + run_step.dependOn(&run_cmd.step); +} diff --git a/linker/tests/fib/platform/host.zig b/linker/tests/fib/platform/host.zig new file mode 100644 index 0000000000..c439c889c7 --- /dev/null +++ b/linker/tests/fib/platform/host.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +extern fn malloc(size: usize) callconv(.C) ?*c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + return malloc(size); +} + +export fn roc_realloc(c_ptr: *c_void, old_size: usize, new_size: usize, alignment: u32) callconv(.C) ?*c_void { + return realloc(@alignCast(16, @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + free(@alignCast(16, @ptrCast([*]u8, c_ptr))); +} + +export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { + const stderr = std.io.getStdErr().writer(); + const msg = @ptrCast([*:0]const u8, c_ptr); + stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; + std.process.exit(0); +} + +const mem = std.mem; +const Allocator = mem.Allocator; + +extern fn roc__mainForHost_1_exposed(i64, *i64) void; + +const Unit = extern struct {}; + +pub export fn main() u8 { + const stdout = std.io.getStdOut().writer(); + const fib_number_to_find: u64 = 10; // find the nth Fibonacci number + const iterations: usize = 50; // number of times to repeatedly find that Fibonacci number + + // make space for the result + var callresult = 0; + var remaining_iterations = iterations; + + while (remaining_iterations > 0) { + // actually call roc to populate the callresult + roc__mainForHost_1_exposed(fib_number_to_find, &callresult); + + remaining_iterations -= 1; + } + + // stdout the final result + stdout.print( + "After calling the Roc app {d} times, the Fibonacci number at index {d} is {d}\n", + .{ iterations, fib_number_to_find, callresult }, + ) catch unreachable; + + return 0; +} diff --git a/roc_std/src/lib.rs b/roc_std/src/lib.rs index d5a15b4627..b6f07dcb8a 100644 --- a/roc_std/src/lib.rs +++ b/roc_std/src/lib.rs @@ -758,69 +758,6 @@ pub enum RocResult { Ok(Ok), } -#[allow(non_camel_case_types)] -type c_char = u8; - -#[repr(u64)] -pub enum RocCallResult { - Success(T), - Failure(*mut c_char), -} - -impl From> for Result { - fn from(call_result: RocCallResult) -> Self { - use RocCallResult::*; - - match call_result { - Success(value) => Ok(value), - Failure(failure) => Err({ - let msg = unsafe { - let mut null_byte_index = 0; - loop { - if *failure.offset(null_byte_index) == 0 { - break; - } - null_byte_index += 1; - } - - let bytes = core::slice::from_raw_parts(failure, null_byte_index as usize); - - core::str::from_utf8_unchecked(bytes) - }; - - msg - }), - } - } -} - -impl<'a, T: Sized + Copy> From<&'a RocCallResult> for Result { - fn from(call_result: &'a RocCallResult) -> Self { - use RocCallResult::*; - - match call_result { - Success(value) => Ok(*value), - Failure(failure) => Err({ - let msg = unsafe { - let mut null_byte_index = 0; - loop { - if *failure.offset(null_byte_index) == 0 { - break; - } - null_byte_index += 1; - } - - let bytes = core::slice::from_raw_parts(*failure, null_byte_index as usize); - - core::str::from_utf8_unchecked(bytes) - }; - - msg - }), - } - } -} - #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub struct RocDec(pub i128); diff --git a/shell.nix b/shell.nix index 0cc46a5998..cfbf59f3fe 100644 --- a/shell.nix +++ b/shell.nix @@ -46,6 +46,7 @@ let python3 llvmPkgs.llvm.dev llvmPkgs.clang + libxkbcommon pkg-config zig