diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 512d4113fc..a7e7ac0ffe 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -7,6 +7,7 @@ name: Benchmarks env: RUST_BACKTRACE: 1 + ROC_NUM_WORKERS: 1 jobs: prep-dependency-container: diff --git a/.gitignore b/.gitignore index 1ce4f54edd..1c7ea24a83 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ generated-docs zig-cache .direnv *.rs.bk +*.o # llvm human-readable output *.ll diff --git a/BUILDING_FROM_SOURCE.md b/BUILDING_FROM_SOURCE.md index 3969d093b4..07b503cd34 100644 --- a/BUILDING_FROM_SOURCE.md +++ b/BUILDING_FROM_SOURCE.md @@ -7,6 +7,7 @@ To build the compiler, you need these installed: * Python 2.7 (Windows only), `python-is-python3` (Ubuntu) * [Zig](https://ziglang.org/), see below for version +* `libxkbcommon` - macOS seems to have it already; on Ubuntu or Debian you can get it with `apt-get install libxkbcommon-dev` * LLVM, see below for version To run the test suite (via `cargo test`), you additionally need to install: @@ -73,6 +74,8 @@ There are also alternative installation options at http://releases.llvm.org/down ## Using Nix +:exclamation: **Our Nix setup is currently broken, you'll have to install manually for now** :exclamation: + ### Install Using [nix](https://nixos.org/download.html) is a quick way to get an environment bootstrapped with a single command. diff --git a/Cargo.lock b/Cargo.lock index 5a26a1d7d0..e8289983d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1720,6 +1720,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "iced-x86" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7383772b06135cede839b7270023b46403656a9148024886e721e82639d3f90e" +dependencies = [ + "lazy_static", + "static_assertions", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -2147,6 +2157,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b6c2ebff6180198788f5db08d7ce3bc1d0b617176678831a7510825973e357" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.5.6" @@ -2498,6 +2517,9 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" dependencies = [ + "crc32fast", + "flate2", + "indexmap", "memchr", ] @@ -3380,7 +3402,9 @@ dependencies = [ "roc_can", "roc_collections", "roc_constrain", + "roc_gen_dev", "roc_gen_llvm", + "roc_gen_wasm", "roc_load", "roc_module", "roc_mono", @@ -3463,6 +3487,7 @@ dependencies = [ "roc_editor", "roc_fmt", "roc_gen_llvm", + "roc_linker", "roc_load", "roc_module", "roc_mono", @@ -3695,6 +3720,24 @@ dependencies = [ name = "roc_ident" version = "0.1.0" +[[package]] +name = "roc_linker" +version = "0.1.0" +dependencies = [ + "bincode", + "bumpalo", + "clap 3.0.0-beta.1", + "iced-x86", + "memmap2 0.3.1", + "object 0.26.2", + "roc_build", + "roc_collections", + "roc_mono", + "serde", + "target-lexicon", + "tempfile", +] + [[package]] name = "roc_load" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 27242439f3..7363ba8600 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "cli/cli_utils", "roc_std", "docs", + "linker", ] exclude = [ "ci/bench-runner" ] # Needed to be able to run `cargo run -p roc_cli --no-default-features` - diff --git a/Earthfile b/Earthfile index 3073021b6b..c6b754ce64 100644 --- a/Earthfile +++ b/Earthfile @@ -46,7 +46,7 @@ install-zig-llvm-valgrind-clippy-rustfmt: copy-dirs: FROM +install-zig-llvm-valgrind-clippy-rustfmt - COPY --dir cli compiler docs editor roc_std vendor examples Cargo.toml Cargo.lock ./ + COPY --dir cli compiler docs editor roc_std vendor examples linker Cargo.toml Cargo.lock ./ test-zig: FROM +install-zig-llvm-valgrind-clippy-rustfmt @@ -66,7 +66,7 @@ check-rustfmt: check-typos: RUN cargo install --version 1.0.11 typos-cli - COPY --dir .github ci cli compiler docs editor examples nightly_benches packages roc_std www *.md LEGAL_DETAILS shell.nix ./ + COPY --dir .github ci cli compiler docs editor examples linker nightly_benches packages roc_std www *.md LEGAL_DETAILS shell.nix ./ RUN typos test-rust: diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0225a11916..97ecb81a46 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false } roc_fmt = { path = "../compiler/fmt" } roc_reporting = { path = "../compiler/reporting" } roc_editor = { path = "../editor", optional = true } +roc_linker = { path = "../linker" } # TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } const_format = "0.2" diff --git a/cli/src/build.rs b/cli/src/build.rs index b9ae821652..85fc609717 100644 --- a/cli/src/build.rs +++ b/cli/src/build.rs @@ -3,6 +3,7 @@ use roc_build::{ link::{link, rebuild_host, LinkType}, program, }; +use roc_builtins::bitcode; use roc_can::builtins::builtin_defs_map; use roc_collections::all::MutMap; use roc_load::file::LoadingProblem; @@ -43,6 +44,7 @@ pub struct BuiltFile { } #[cfg(feature = "llvm")] +#[allow(clippy::too_many_arguments)] pub fn build_file<'a>( arena: &'a Bump, target: &Triple, @@ -50,7 +52,10 @@ pub fn build_file<'a>( roc_file_path: PathBuf, opt_level: OptLevel, emit_debug_info: bool, + emit_timings: bool, link_type: LinkType, + surgically_link: bool, + precompiled: bool, ) -> Result> { let compilation_start = SystemTime::now(); let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; @@ -59,10 +64,7 @@ pub fn build_file<'a>( let subs_by_module = MutMap::default(); // Release builds use uniqueness optimizations - let stdlib = match opt_level { - OptLevel::Normal => arena.alloc(roc_builtins::std::standard_stdlib()), - OptLevel::Optimize => arena.alloc(roc_builtins::std::standard_stdlib()), - }; + let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); let loaded = roc_load::file::load_and_monomorphize( arena, @@ -75,16 +77,7 @@ pub fn build_file<'a>( )?; use target_lexicon::Architecture; - let emit_wasm = match target.architecture { - Architecture::X86_64 => false, - Architecture::Aarch64(_) => false, - Architecture::Wasm32 => true, - Architecture::X86_32(_) => false, - _ => panic!( - "TODO gracefully handle unsupported architecture: {:?}", - target.architecture - ), - }; + let emit_wasm = matches!(target.architecture, Architecture::Wasm32); // TODO wasm host extension should be something else ideally // .bc does not seem to work because @@ -95,7 +88,39 @@ pub fn build_file<'a>( let host_extension = if emit_wasm { "zig" } else { "o" }; let app_extension = if emit_wasm { "bc" } else { "o" }; + let cwd = roc_file_path.parent().unwrap(); + let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows + + if emit_wasm { + binary_path.set_extension("wasm"); + } + + let mut host_input_path = PathBuf::from(cwd); let path_to_platform = loaded.platform_path.clone(); + host_input_path.push(&*path_to_platform); + host_input_path.push("host"); + host_input_path.set_extension(host_extension); + + // TODO this should probably be moved before load_and_monomorphize. + // To do this we will need to preprocess files just for their exported symbols. + // Also, we should no longer need to do this once we have platforms on + // a package repository, as we can then get precompiled hosts from there. + let rebuild_thread = spawn_rebuild_thread( + opt_level, + surgically_link, + precompiled, + host_input_path.clone(), + binary_path.clone(), + target, + loaded + .exposed_to_host + .keys() + .map(|x| x.as_str(&loaded.interns).to_string()) + .collect(), + ); + + // TODO try to move as much of this linking as possible to the precompiled + // host, to minimize the amount of host-application linking required. let app_o_file = Builder::new() .prefix("roc_app") .suffix(&format!(".{}", app_extension)) @@ -146,17 +171,26 @@ pub fn build_file<'a>( } } - let cwd = roc_file_path.parent().unwrap(); - let binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows - let code_gen_timing = program::gen_from_mono_module( - arena, - loaded, - &roc_file_path, - target, - app_o_file, - opt_level, - emit_debug_info, - ); + // This only needs to be mutable for report_problems. This can't be done + // inside a nested scope without causing a borrow error! + let mut loaded = loaded; + program::report_problems(&mut loaded); + let loaded = loaded; + + let code_gen_timing = match opt_level { + OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm( + arena, + loaded, + &roc_file_path, + target, + app_o_file, + opt_level, + emit_debug_info, + ), + OptLevel::Development => { + program::gen_from_mono_module_dev(arena, loaded, target, app_o_file) + } + }; buf.push('\n'); buf.push_str(" "); @@ -177,7 +211,7 @@ pub fn build_file<'a>( }) .len(); - if emit_debug_info { + if emit_timings { println!( "\n\nCompilation finished!\n\nHere's how long each module took to compile:\n\n{}", buf @@ -190,65 +224,189 @@ pub fn build_file<'a>( ); } - // Step 2: link the precompiled host and compiled app - let mut host_input_path = PathBuf::from(cwd); - - host_input_path.push(&*path_to_platform); - host_input_path.push("host"); - host_input_path.set_extension(host_extension); - - // TODO we should no longer need to do this once we have platforms on - // a package repository, as we can then get precompiled hosts from there. - let rebuild_host_start = SystemTime::now(); - rebuild_host(target, host_input_path.as_path()); - let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); - - if emit_debug_info { + let rebuild_duration = rebuild_thread.join().unwrap(); + if emit_timings && !precompiled { println!( - "Finished rebuilding the host in {} ms\n", - rebuild_host_end.as_millis() + "Finished rebuilding and preprocessing the host in {} ms\n", + rebuild_duration ); } - // TODO try to move as much of this linking as possible to the precompiled - // host, to minimize the amount of host-application linking required. + // Step 2: link the precompiled host and compiled app let link_start = SystemTime::now(); - let (mut child, binary_path) = // TODO use lld - link( - target, - binary_path, - &[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()], - link_type - ) - .map_err(|_| { - todo!("gracefully handle `rustc` failing to spawn."); + let outcome = if surgically_link { + roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path) + .map_err(|_| { + todo!("gracefully handle failing to surgically link"); + })?; + BuildOutcome::NoProblems + } else { + let mut inputs = vec![ + host_input_path.as_path().to_str().unwrap(), + app_o_file.to_str().unwrap(), + ]; + if matches!(opt_level, OptLevel::Development) { + inputs.push(bitcode::OBJ_PATH); + } + + let (mut child, _) = // TODO use lld + link( + target, + binary_path.clone(), + &inputs, + link_type + ) + .map_err(|_| { + todo!("gracefully handle `ld` failing to spawn."); + })?; + + let exit_status = child.wait().map_err(|_| { + todo!("gracefully handle error after `ld` spawned"); })?; - let cmd_result = child.wait().map_err(|_| { - todo!("gracefully handle error after `rustc` spawned"); - }); - + // TODO change this to report whether there were errors or warnings! + if exit_status.success() { + BuildOutcome::NoProblems + } else { + BuildOutcome::Errors + } + }; let linking_time = link_start.elapsed().unwrap(); - if emit_debug_info { + if emit_timings { println!("Finished linking in {} ms\n", linking_time.as_millis()); } let total_time = compilation_start.elapsed().unwrap(); - // If the cmd errored out, return the Err. - let exit_status = cmd_result?; - - // TODO change this to report whether there were errors or warnings! - let outcome = if exit_status.success() { - BuildOutcome::NoProblems - } else { - BuildOutcome::Errors - }; - Ok(BuiltFile { binary_path, outcome, total_time, }) } + +fn spawn_rebuild_thread( + opt_level: OptLevel, + surgically_link: bool, + precompiled: bool, + host_input_path: PathBuf, + binary_path: PathBuf, + target: &Triple, + exported_symbols: Vec, +) -> std::thread::JoinHandle { + let thread_local_target = target.clone(); + std::thread::spawn(move || { + let rebuild_host_start = SystemTime::now(); + if !precompiled { + if surgically_link { + roc_linker::build_and_preprocess_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + exported_symbols, + ) + .unwrap(); + } else { + rebuild_host( + opt_level, + &thread_local_target, + host_input_path.as_path(), + None, + ); + } + } + if surgically_link { + // Copy preprocessed host to executable location. + let prehost = host_input_path.with_file_name("preprocessedhost"); + std::fs::copy(prehost, binary_path.as_path()).unwrap(); + } + let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); + rebuild_host_end.as_millis() + }) +} + +#[allow(clippy::too_many_arguments)] +pub fn check_file( + arena: &Bump, + src_dir: PathBuf, + roc_file_path: PathBuf, + emit_timings: bool, +) -> Result { + let compilation_start = SystemTime::now(); + + // only used for generating errors. We don't do code generation, so hardcoding should be fine + // we need monomorphization for when exhaustiveness checking + let ptr_bytes = 8; + + // Step 1: compile the app and generate the .o file + let subs_by_module = MutMap::default(); + + // Release builds use uniqueness optimizations + let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); + + let mut loaded = roc_load::file::load_and_monomorphize( + arena, + roc_file_path, + stdlib, + src_dir.as_path(), + subs_by_module, + ptr_bytes, + builtin_defs_map, + )?; + + let buf = &mut String::with_capacity(1024); + + let mut it = loaded.timings.iter().peekable(); + while let Some((module_id, module_timing)) = it.next() { + let module_name = loaded.interns.module_name(*module_id); + + buf.push_str(" "); + + if module_name.is_empty() { + // the App module + buf.push_str("Application Module"); + } else { + buf.push_str(module_name); + } + + buf.push('\n'); + + report_timing(buf, "Read .roc file from disk", module_timing.read_roc_file); + report_timing(buf, "Parse header", module_timing.parse_header); + report_timing(buf, "Parse body", module_timing.parse_body); + report_timing(buf, "Canonicalize", module_timing.canonicalize); + report_timing(buf, "Constrain", module_timing.constrain); + report_timing(buf, "Solve", module_timing.solve); + report_timing( + buf, + "Find Specializations", + module_timing.find_specializations, + ); + report_timing( + buf, + "Make Specializations", + module_timing.make_specializations, + ); + report_timing(buf, "Other", module_timing.other()); + buf.push('\n'); + report_timing(buf, "Total", module_timing.total()); + + if it.peek().is_some() { + buf.push('\n'); + } + } + + let compilation_end = compilation_start.elapsed().unwrap(); + + if emit_timings { + println!( + "\n\nCompilation finished!\n\nHere's how long each module took to compile:\n\n{}", + buf + ); + + println!("Finished checking in {} ms\n", compilation_end.as_millis(),); + } + + Ok(program::report_problems(&mut loaded)) +} diff --git a/cli/src/lib.rs b/cli/src/lib.rs index 678b613737..c7b14a56f8 100644 --- a/cli/src/lib.rs +++ b/cli/src/lib.rs @@ -26,11 +26,16 @@ pub const CMD_BUILD: &str = "build"; pub const CMD_REPL: &str = "repl"; pub const CMD_EDIT: &str = "edit"; pub const CMD_DOCS: &str = "docs"; +pub const CMD_CHECK: &str = "check"; pub const FLAG_DEBUG: &str = "debug"; +pub const FLAG_DEV: &str = "dev"; pub const FLAG_OPTIMIZE: &str = "optimize"; pub const FLAG_LIB: &str = "lib"; pub const FLAG_BACKEND: &str = "backend"; +pub const FLAG_TIME: &str = "time"; +pub const FLAG_LINK: &str = "roc-linker"; +pub const FLAG_PRECOMPILED: &str = "precompiled-host"; pub const ROC_FILE: &str = "ROC_FILE"; pub const BACKEND: &str = "BACKEND"; pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES"; @@ -53,6 +58,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Optimize your compiled Roc program to run faster. (Optimization takes time to complete.)") .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -74,6 +85,24 @@ pub fn build_app<'a>() -> App<'a> { .help("Store LLVM debug information in the generated program") .required(false), ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) ) .subcommand(App::new(CMD_RUN) .about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`") @@ -84,6 +113,12 @@ pub fn build_app<'a>() -> App<'a> { .help("Optimize the compiled program to run faster. (Optimization takes time to complete.)") .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_DEBUG) .long(FLAG_DEBUG) @@ -104,6 +139,20 @@ pub fn build_app<'a>() -> App<'a> { .subcommand(App::new(CMD_REPL) .about("Launch the interactive Read Eval Print Loop (REPL)") ) + .subcommand(App::new(CMD_CHECK) + .about("Build a binary from the given .roc file, but don't run it") + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(ROC_FILE) + .help("The .roc file of an app to run") + .required(true), + ) + ) .subcommand( App::new(CMD_DOCS) .about("Generate documentation for Roc modules") @@ -123,6 +172,12 @@ pub fn build_app<'a>() -> App<'a> { .requires(ROC_FILE) .required(false), ) + .arg( + Arg::with_name(FLAG_DEV) + .long(FLAG_DEV) + .help("Make compilation as fast as possible. (Runtime performance may suffer)") + .required(false), + ) .arg( Arg::with_name(FLAG_DEBUG) .long(FLAG_DEBUG) @@ -130,6 +185,24 @@ pub fn build_app<'a>() -> App<'a> { .requires(ROC_FILE) .required(false), ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .help("Prints detailed compilation time information.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_LINK) + .long(FLAG_LINK) + .help("Uses the roc linker instead of the system linker.") + .required(false), + ) + .arg( + Arg::with_name(FLAG_PRECOMPILED) + .long(FLAG_PRECOMPILED) + .help("Assumes the host has been precompiled and skips recompiling the host.") + .required(false), + ) .arg( Arg::with_name(FLAG_BACKEND) .long(FLAG_BACKEND) @@ -197,18 +270,31 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { let filename = matches.value_of(ROC_FILE).unwrap(); let original_cwd = std::env::current_dir()?; - let opt_level = if matches.is_present(FLAG_OPTIMIZE) { - OptLevel::Optimize - } else { - OptLevel::Normal + let opt_level = match ( + matches.is_present(FLAG_OPTIMIZE), + matches.is_present(FLAG_DEV), + ) { + (true, false) => OptLevel::Optimize, + (true, true) => panic!("development cannot be optimized!"), + (false, true) => OptLevel::Development, + (false, false) => OptLevel::Normal, }; let emit_debug_info = matches.is_present(FLAG_DEBUG); + let emit_timings = matches.is_present(FLAG_TIME); let link_type = if matches.is_present(FLAG_LIB) { LinkType::Dylib } else { LinkType::Executable }; + let surgically_link = matches.is_present(FLAG_LINK); + let precompiled = matches.is_present(FLAG_PRECOMPILED); + if surgically_link && !roc_linker::supported(&link_type, &target) { + panic!( + "Link type, {:?}, with target, {}, not supported by roc linker", + link_type, target + ); + } let path = Path::new(filename); @@ -239,7 +325,10 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result { path, opt_level, emit_debug_info, + emit_timings, link_type, + surgically_link, + precompiled, ); match res_binary_path { @@ -393,9 +482,7 @@ enum Backend { Host, X86_32, X86_64, - Dev, Wasm32, - Wasm32Dev, } impl Default for Backend { @@ -410,9 +497,7 @@ impl Backend { Backend::Host => "host", Backend::X86_32 => "x86_32", Backend::X86_64 => "x86_64", - Backend::Dev => "dev", Backend::Wasm32 => "wasm32", - Backend::Wasm32Dev => "wasm32_dev", } } @@ -421,9 +506,7 @@ impl Backend { Backend::Host.as_str(), Backend::X86_32.as_str(), Backend::X86_64.as_str(), - Backend::Dev.as_str(), Backend::Wasm32.as_str(), - Backend::Wasm32Dev.as_str(), ]; fn to_triple(&self) -> Triple { @@ -446,8 +529,7 @@ impl Backend { triple } - Backend::Dev => todo!(), - Backend::Wasm32 | Backend::Wasm32Dev => { + Backend::Wasm32 => { triple.architecture = Architecture::Wasm32; triple.binary_format = BinaryFormat::Wasm; @@ -471,9 +553,7 @@ impl std::str::FromStr for Backend { "host" => Ok(Backend::Host), "x86_32" => Ok(Backend::X86_32), "x86_64" => Ok(Backend::X86_64), - "dev" => Ok(Backend::Dev), "wasm32" => Ok(Backend::Wasm32), - "wasm32_dev" => Ok(Backend::Wasm32Dev), _ => Err(()), } } diff --git a/cli/src/main.rs b/cli/src/main.rs index 5a699a3f57..739b341907 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,7 +1,9 @@ +use roc_cli::build::check_file; use roc_cli::{ - build_app, docs, repl, BuildConfig, CMD_BUILD, CMD_DOCS, CMD_EDIT, CMD_REPL, CMD_RUN, - DIRECTORY_OR_FILES, ROC_FILE, + build_app, docs, repl, BuildConfig, CMD_BUILD, CMD_CHECK, CMD_DOCS, CMD_EDIT, CMD_REPL, + CMD_RUN, DIRECTORY_OR_FILES, FLAG_TIME, ROC_FILE, }; +use roc_load::file::LoadingProblem; use std::fs::{self, FileType}; use std::io; use std::path::{Path, PathBuf}; @@ -52,6 +54,31 @@ If you're building the compiler from source you'll want to do `cargo run [FILE]` Ok(1) } + Some(CMD_CHECK) => { + let arena = bumpalo::Bump::new(); + + let matches = matches.subcommand_matches(CMD_CHECK).unwrap(); + let emit_timings = matches.is_present(FLAG_TIME); + let filename = matches.value_of(ROC_FILE).unwrap(); + let roc_file_path = PathBuf::from(filename); + let src_dir = roc_file_path.parent().unwrap().to_owned(); + + match check_file(&arena, src_dir, roc_file_path, emit_timings) { + Ok(number_of_errors) => { + let exit_code = if number_of_errors != 0 { 1 } else { 0 }; + Ok(exit_code) + } + + Err(LoadingProblem::FormattedReport(report)) => { + print!("{}", report); + + Ok(1) + } + Err(other) => { + panic!("build_file failed with error:\n{:?}", other); + } + } + } Some(CMD_REPL) => { repl::main()?; diff --git a/cli/src/repl/eval.rs b/cli/src/repl/eval.rs index c3792a1153..882cf89c22 100644 --- a/cli/src/repl/eval.rs +++ b/cli/src/repl/eval.rs @@ -353,6 +353,13 @@ fn jit_to_ast_help<'a>( | Layout::RecursivePointer => { todo!("add support for rendering recursive tag unions in the REPL") } + Layout::LambdaSet(lambda_set) => jit_to_ast_help( + env, + lib, + main_fn_name, + &lambda_set.runtime_representation(), + content, + ), } } diff --git a/cli/src/repl/gen.rs b/cli/src/repl/gen.rs index f8cb2acab6..200941919d 100644 --- a/cli/src/repl/gen.rs +++ b/cli/src/repl/gen.rs @@ -107,12 +107,13 @@ pub fn gen_and_eval<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/cli/tests/cli_run.rs b/cli/tests/cli_run.rs index 73aa75880c..aa7020700f 100644 --- a/cli/tests/cli_run.rs +++ b/cli/tests/cli_run.rs @@ -157,6 +157,15 @@ mod cli_run { let example = $example; let file_name = example_file(dir_name, example.filename); + match example.executable_filename { + "hello-web" => { + // this is a web webassembly example, but we don't test with JS at the moment + eprintln!("WARNING: skipping testing example {} because the test is broken right now!", example.filename); + return; + } + _ => {} + } + // Check with and without optimizations check_output_with_stdin( &file_name, @@ -224,6 +233,20 @@ mod cli_run { expected_ending:"Hello, World!\n", use_valgrind: true, }, + hello_web:"hello-web" => Example { + filename: "Hello.roc", + executable_filename: "hello-web", + stdin: &[], + expected_ending:"Hello, World!\n", + use_valgrind: true, + }, + fib:"fib" => Example { + filename: "Fib.roc", + executable_filename: "fib", + stdin: &[], + expected_ending:"55\n", + use_valgrind: true, + }, quicksort:"quicksort" => Example { filename: "Quicksort.roc", executable_filename: "quicksort", @@ -242,7 +265,7 @@ mod cli_run { filename: "Main.roc", executable_filename: "effect-example", stdin: &["hi there!"], - expected_ending: "hi there!\n", + expected_ending: "hi there!\nIt is known\n", use_valgrind: true, }, // tea:"tea" => Example { diff --git a/cli/tests/fixtures/multi-dep-str/platform/host.zig b/cli/tests/fixtures/multi-dep-str/platform/host.zig index bf16bd9b65..f72ded7353 100644 --- a/cli/tests/fixtures/multi-dep-str/platform/host.zig +++ b/cli/tests/fixtures/multi-dep-str/platform/host.zig @@ -22,7 +22,8 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; +extern fn roc__mainForHost_1_exposed() RocStr; + extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; @@ -47,28 +48,23 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } -const RocCallResult = extern struct { flag: usize, content: RocStr }; - const Unit = extern struct {}; pub export fn main() i32 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + const callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/cli/tests/fixtures/multi-dep-thunk/platform/host.zig b/cli/tests/fixtures/multi-dep-thunk/platform/host.zig index bf16bd9b65..4c8ee671cf 100644 --- a/cli/tests/fixtures/multi-dep-thunk/platform/host.zig +++ b/cli/tests/fixtures/multi-dep-thunk/platform/host.zig @@ -22,7 +22,7 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; +extern fn roc__mainForHost_1_exposed() RocStr; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; @@ -47,28 +47,23 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } -const RocCallResult = extern struct { flag: usize, content: RocStr }; - const Unit = extern struct {}; pub export fn main() i32 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + const callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/compiler/build/Cargo.toml b/compiler/build/Cargo.toml index c49608fd97..decaa91328 100644 --- a/compiler/build/Cargo.toml +++ b/compiler/build/Cargo.toml @@ -20,6 +20,8 @@ roc_solve = { path = "../solve" } roc_mono = { path = "../mono" } roc_load = { path = "../load" } roc_gen_llvm = { path = "../gen_llvm", optional = true } +roc_gen_wasm = { path = "../gen_wasm" } +roc_gen_dev = { path = "../gen_dev" } roc_reporting = { path = "../reporting" } roc_std = { path = "../../roc_std" } im = "14" # im and im-rc should always have the same version! diff --git a/compiler/build/src/link.rs b/compiler/build/src/link.rs index b1fd76c1ee..2d3517ee60 100644 --- a/compiler/build/src/link.rs +++ b/compiler/build/src/link.rs @@ -1,6 +1,7 @@ use crate::target::arch_str; #[cfg(feature = "llvm")] use libloading::{Error, Library}; +use roc_builtins::bitcode; #[cfg(feature = "llvm")] use roc_mono::ir::OptLevel; use std::collections::HashMap; @@ -56,7 +57,7 @@ fn find_zig_str_path() -> PathBuf { return zig_str_path; } - panic!("cannot find `str.zig`") + panic!("cannot find `str.zig`. Launch me from either the root of the roc repo or one level down(roc/examples, roc/cli...)") } fn find_wasi_libc_path() -> PathBuf { @@ -76,6 +77,7 @@ fn find_wasi_libc_path() -> PathBuf { } #[cfg(not(target_os = "macos"))] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -83,33 +85,48 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) - .env("HOME", env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "-fcompiler-rt", - // include libc - "--library", - "c", - // cross-compile? - "-target", - target, - ]) - .output() - .unwrap() + .env("HOME", env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + "build-exe", + "-fPIE", + shared_lib_path.to_str().unwrap(), + bitcode::OBJ_PATH, + ]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "-fcompiler-rt", + // include libc + "--library", + "c", + // cross-compile? + "-target", + target, + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } #[cfg(target_os = "macos")] +#[allow(clippy::too_many_arguments)] pub fn build_zig_host_native( env_path: &str, env_home: &str, @@ -117,6 +134,8 @@ pub fn build_zig_host_native( zig_host_src: &str, zig_str_path: &str, _target: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { use serde_json::Value; @@ -158,29 +177,41 @@ pub fn build_zig_host_native( zig_compiler_rt_path.push("special"); zig_compiler_rt_path.push("compiler_rt.zig"); - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", &env_path) - .env("HOME", &env_home) - .args(&[ - "build-obj", - zig_host_src, - emit_bin, - "--pkg-begin", - "str", - zig_str_path, - "--pkg-end", - // include the zig runtime - "--pkg-begin", - "compiler_rt", - zig_compiler_rt_path.to_str().unwrap(), - "--pkg-end", - // include libc - "--library", - "c", - ]) - .output() - .unwrap() + .env("HOME", &env_home); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + "build-exe", + "-fPIE", + shared_lib_path.to_str().unwrap(), + bitcode::OBJ_PATH, + ]); + } else { + command.args(&["build-obj", "-fPIC"]); + } + command.args(&[ + zig_host_src, + emit_bin, + "--pkg-begin", + "str", + zig_str_path, + "--pkg-end", + // include the zig runtime + "--pkg-begin", + "compiler_rt", + zig_compiler_rt_path.to_str().unwrap(), + "--pkg-end", + // include libc + "--library", + "c", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } pub fn build_zig_host_wasm32( @@ -189,7 +220,12 @@ pub fn build_zig_host_wasm32( emit_bin: &str, zig_host_src: &str, zig_str_path: &str, + opt_level: OptLevel, + shared_lib_path: Option<&Path>, ) -> Output { + if shared_lib_path.is_some() { + unimplemented!("Linking a shared library to wasm not yet implemented"); + } // NOTE currently just to get compiler warnings if the host code is invalid. // the produced artifact is not used // @@ -198,7 +234,8 @@ pub fn build_zig_host_wasm32( // we'd like to compile with `-target wasm32-wasi` but that is blocked on // // https://github.com/ziglang/zig/issues/9414 - Command::new("zig") + let mut command = Command::new("zig"); + command .env_clear() .env("PATH", env_path) .env("HOME", env_home) @@ -219,19 +256,68 @@ pub fn build_zig_host_wasm32( "i386-linux-musl", // "wasm32-wasi", // "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll", - ]) - .output() - .unwrap() + "-fPIC", + "--strip", + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.args(&["-O", "ReleaseSafe"]); + } + command.output().unwrap() } -pub fn rebuild_host(target: &Triple, host_input_path: &Path) { +pub fn build_c_host_native( + env_path: &str, + env_home: &str, + dest: &str, + sources: &[&str], + opt_level: OptLevel, + shared_lib_path: Option<&Path>, +) -> Output { + let mut command = Command::new("clang"); + command + .env_clear() + .env("PATH", &env_path) + .env("HOME", &env_home) + .args(sources) + .args(&["-o", dest]); + if let Some(shared_lib_path) = shared_lib_path { + command.args(&[ + shared_lib_path.to_str().unwrap(), + bitcode::OBJ_PATH, + "-fPIE", + "-pie", + "-lm", + "-lpthread", + "-ldl", + "-lrt", + "-lutil", + ]); + } else { + command.args(&["-fPIC", "-c"]); + } + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O2"); + } + command.output().unwrap() +} + +pub fn rebuild_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + shared_lib_path: Option<&Path>, +) { let c_host_src = host_input_path.with_file_name("host.c"); let c_host_dest = host_input_path.with_file_name("c_host.o"); let zig_host_src = host_input_path.with_file_name("host.zig"); let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); - let host_dest_native = host_input_path.with_file_name("host.o"); + let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() { + "dynhost" + } else { + "host.o" + }); let host_dest_wasm = host_input_path.with_file_name("host.bc"); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); @@ -257,6 +343,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { &emit_bin, zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), + opt_level, + shared_lib_path, ) } Architecture::X86_64 => { @@ -268,6 +356,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "native", + opt_level, + shared_lib_path, ) } Architecture::X86_32(_) => { @@ -279,87 +369,142 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { zig_host_src.to_str().unwrap(), zig_str_path.to_str().unwrap(), "i386-linux-musl", + opt_level, + shared_lib_path, ) } _ => panic!("Unsupported architecture {:?}", target.architecture), }; validate_output("host.zig", "zig", output) - } else { - // Compile host.c - let output = Command::new("clang") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-c", - c_host_src.to_str().unwrap(), - "-o", - c_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); - - validate_output("host.c", "clang", output); - } - - if cargo_host_src.exists() { + } else if cargo_host_src.exists() { // Compile and link Cargo.toml, if it exists let cargo_dir = host_input_path.parent().unwrap(); - let libhost_dir = cargo_dir.join("target").join("release"); + let libhost_dir = + cargo_dir + .join("target") + .join(if matches!(opt_level, OptLevel::Optimize) { + "release" + } else { + "debug" + }); + let libhost = libhost_dir.join("libhost.a"); - let output = Command::new("cargo") - .args(&["build", "--release"]) - .current_dir(cargo_dir) - .output() - .unwrap(); + let mut command = Command::new("cargo"); + command.arg("build").current_dir(cargo_dir); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("--release"); + } + let output = command.output().unwrap(); - validate_output("src/lib.rs", "cargo build --release", output); + validate_output("src/lib.rs", "cargo build", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - "-L", - libhost_dir.to_str().unwrap(), - c_host_dest.to_str().unwrap(), - "-lhost", - "-o", + // Cargo hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); - validate_output("c_host.o", "ld", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + "-L", + libhost_dir.to_str().unwrap(), + c_host_dest.to_str().unwrap(), + "-lhost", + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); + validate_output("c_host.o", "ld", output); + + // Clean up c_host.o + let output = Command::new("rm") + .env_clear() + .args(&["-f", c_host_dest.to_str().unwrap()]) + .output() + .unwrap(); + + validate_output("rust_host.o", "rm", output); + } } else if rust_host_src.exists() { // Compile and link host.rs, if it exists - let output = Command::new("rustc") - .args(&[ - rust_host_src.to_str().unwrap(), - "-o", - rust_host_dest.to_str().unwrap(), - ]) - .output() - .unwrap(); + let mut command = Command::new("rustc"); + command.args(&[ + rust_host_src.to_str().unwrap(), + "-o", + rust_host_dest.to_str().unwrap(), + ]); + if matches!(opt_level, OptLevel::Optimize) { + command.arg("-O"); + } + let output = command.output().unwrap(); validate_output("host.rs", "rustc", output); - let output = Command::new("ld") - .env_clear() - .env("PATH", &env_path) - .args(&[ - "-r", - c_host_dest.to_str().unwrap(), - rust_host_dest.to_str().unwrap(), - "-o", + // Rust hosts depend on a c wrapper for the api. Compile host.c as well. + if shared_lib_path.is_some() { + // If compiling to executable, let c deal with linking as well. + let output = build_c_host_native( + &env_path, + &env_home, host_dest_native.to_str().unwrap(), - ]) - .output() - .unwrap(); + &[ + c_host_src.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + ], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); + } else { + let output = build_c_host_native( + &env_path, + &env_home, + c_host_dest.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); - validate_output("rust_host.o", "ld", output); + validate_output("host.c", "clang", output); + let output = Command::new("ld") + .env_clear() + .env("PATH", &env_path) + .args(&[ + "-r", + c_host_dest.to_str().unwrap(), + rust_host_dest.to_str().unwrap(), + "-o", + host_dest_native.to_str().unwrap(), + ]) + .output() + .unwrap(); - // Clean up rust_host.o + validate_output("rust_host.o", "ld", output); + } + + // Clean up rust_host.o and c_host.o let output = Command::new("rm") .env_clear() .args(&[ @@ -371,15 +516,17 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) { .unwrap(); validate_output("rust_host.o", "rm", output); - } else if c_host_dest.exists() { - // Clean up c_host.o - let output = Command::new("mv") - .env_clear() - .args(&[c_host_dest, host_dest_native]) - .output() - .unwrap(); - - validate_output("c_host.o", "mv", output); + } else if c_host_src.exists() { + // Compile host.c, if it exists + let output = build_c_host_native( + &env_path, + &env_home, + host_dest_native.to_str().unwrap(), + &[c_host_src.to_str().unwrap()], + opt_level, + shared_lib_path, + ); + validate_output("host.c", "clang", output); } } @@ -521,6 +668,7 @@ fn link_linux( "--eh-frame-hdr", "-arch", arch_str(target), + "-pie", libcrt_path.join("crti.o").to_str().unwrap(), libcrt_path.join("crtn.o").to_str().unwrap(), ]) diff --git a/compiler/build/src/program.rs b/compiler/build/src/program.rs index 178577b3e8..226bfaf8be 100644 --- a/compiler/build/src/program.rs +++ b/compiler/build/src/program.rs @@ -10,6 +10,8 @@ use roc_mono::ir::OptLevel; use std::path::{Path, PathBuf}; use std::time::Duration; +use roc_collections::all::{MutMap, MutSet}; + #[derive(Debug, Clone, Copy, Default)] pub struct CodeGenTiming { pub code_gen: Duration, @@ -20,33 +22,15 @@ pub struct CodeGenTiming { // llvm we're using, consider moving me somewhere else. const LLVM_VERSION: &str = "12"; -// TODO how should imported modules factor into this? What if those use builtins too? -// TODO this should probably use more helper functions -// TODO make this polymorphic in the llvm functions so it can be reused for another backend. -#[cfg(feature = "llvm")] -#[allow(clippy::cognitive_complexity)] -pub fn gen_from_mono_module( - arena: &bumpalo::Bump, - mut loaded: MonomorphizedModule, - roc_file_path: &Path, - target: &target_lexicon::Triple, - app_o_file: &Path, - opt_level: OptLevel, - emit_debug_info: bool, -) -> CodeGenTiming { - use crate::target::{self, convert_opt_level}; - use inkwell::attributes::{Attribute, AttributeLoc}; - use inkwell::context::Context; - use inkwell::module::Linkage; - use inkwell::targets::{CodeModel, FileType, RelocMode}; - use std::time::SystemTime; - +// TODO instead of finding exhaustiveness problems in monomorphization, find +// them after type checking (like Elm does) so we can complete the entire +// `roc check` process without needing to monomorphize. +/// Returns the number of problems reported. +pub fn report_problems(loaded: &mut MonomorphizedModule) -> usize { use roc_reporting::report::{ can_problem, mono_problem, type_problem, Report, RocDocAllocator, Severity::*, DEFAULT_PALETTE, }; - - let code_gen_start = SystemTime::now(); let palette = DEFAULT_PALETTE; // This will often over-allocate total memory, but it means we definitely @@ -55,10 +39,10 @@ pub fn gen_from_mono_module( let mut warnings = Vec::with_capacity(total_problems); let mut errors = Vec::with_capacity(total_problems); - for (home, (module_path, src)) in loaded.sources { + for (home, (module_path, src)) in loaded.sources.iter() { let mut src_lines: Vec<&str> = Vec::new(); - if let Some((_, header_src)) = loaded.header_sources.get(&home) { + if let Some((_, header_src)) = loaded.header_sources.get(home) { src_lines.extend(header_src.split('\n')); src_lines.extend(src.split('\n').skip(1)); } else { @@ -66,9 +50,10 @@ pub fn gen_from_mono_module( } // Report parsing and canonicalization problems - let alloc = RocDocAllocator::new(&src_lines, home, &loaded.interns); + let alloc = RocDocAllocator::new(&src_lines, *home, &loaded.interns); + + let problems = loaded.can_problems.remove(home).unwrap_or_default(); - let problems = loaded.can_problems.remove(&home).unwrap_or_default(); for problem in problems.into_iter() { let report = can_problem(&alloc, module_path.clone(), problem); let severity = report.severity; @@ -86,25 +71,28 @@ pub fn gen_from_mono_module( } } - let problems = loaded.type_problems.remove(&home).unwrap_or_default(); + let problems = loaded.type_problems.remove(home).unwrap_or_default(); + for problem in problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let severity = report.severity; - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let severity = report.severity; + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - match severity { - Warning => { - warnings.push(buf); - } - RuntimeError => { - errors.push(buf); + match severity { + Warning => { + warnings.push(buf); + } + RuntimeError => { + errors.push(buf); + } } } } - let problems = loaded.mono_problems.remove(&home).unwrap_or_default(); + let problems = loaded.mono_problems.remove(home).unwrap_or_default(); + for problem in problems { let report = mono_problem(&alloc, module_path.clone(), problem); let severity = report.severity; @@ -123,12 +111,18 @@ pub fn gen_from_mono_module( } } + let problems_reported; + // Only print warnings if there are no errors if errors.is_empty() { + problems_reported = warnings.len(); + for warning in warnings { println!("\n{}\n", warning); } } else { + problems_reported = errors.len(); + for error in errors { println!("\n{}\n", error); } @@ -140,10 +134,35 @@ pub fn gen_from_mono_module( // The horizontal rule is nice when running the program right after // compiling it, as it lets you clearly see where the compiler // errors/warnings end and the program output begins. - if total_problems > 0 { + if problems_reported > 0 { println!("{}\u{001B}[0m\n", Report::horizontal_rule(&palette)); } + problems_reported +} + +// TODO how should imported modules factor into this? What if those use builtins too? +// TODO this should probably use more helper functions +// TODO make this polymorphic in the llvm functions so it can be reused for another backend. +#[cfg(feature = "llvm")] +pub fn gen_from_mono_module_llvm( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + roc_file_path: &Path, + target: &target_lexicon::Triple, + app_o_file: &Path, + opt_level: OptLevel, + emit_debug_info: bool, +) -> CodeGenTiming { + use crate::target::{self, convert_opt_level}; + use inkwell::attributes::{Attribute, AttributeLoc}; + use inkwell::context::Context; + use inkwell::module::Linkage; + use inkwell::targets::{CodeModel, FileType, RelocMode}; + use std::time::SystemTime; + + let code_gen_start = SystemTime::now(); + // Generate the binary let ptr_bytes = target.pointer_width().unwrap().bytes() as u32; let context = Context::create(); @@ -286,6 +305,7 @@ pub fn gen_from_mono_module( .unwrap(); let llc_args = &[ + "-relocation-model=pic", "-filetype=obj", app_bc_file.to_str().unwrap(), "-o", @@ -325,7 +345,7 @@ pub fn gen_from_mono_module( use target_lexicon::Architecture; match target.architecture { Architecture::X86_64 | Architecture::X86_32(_) | Architecture::Aarch64(_) => { - let reloc = RelocMode::Default; + let reloc = RelocMode::PIC; let model = CodeModel::Default; let target_machine = target::target_machine(target, convert_opt_level(opt_level), reloc, model) @@ -354,3 +374,78 @@ pub fn gen_from_mono_module( emit_o_file, } } + +pub fn gen_from_mono_module_dev( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + target: &target_lexicon::Triple, + app_o_file: &Path, +) -> CodeGenTiming { + use target_lexicon::Architecture; + + match target.architecture { + Architecture::Wasm32 => gen_from_mono_module_dev_wasm32(arena, loaded, app_o_file), + Architecture::X86_64 => { + gen_from_mono_module_dev_assembly(arena, loaded, target, app_o_file) + } + _ => todo!(), + } +} + +fn gen_from_mono_module_dev_wasm32( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + app_o_file: &Path, +) -> CodeGenTiming { + let mut procedures = MutMap::default(); + + for (key, proc) in loaded.procedures { + procedures.insert(key, proc); + } + + let exposed_to_host = loaded + .exposed_to_host + .keys() + .copied() + .collect::>(); + + let env = roc_gen_wasm::Env { + arena, + interns: loaded.interns, + exposed_to_host, + }; + + let bytes = roc_gen_wasm::build_module(&env, procedures).unwrap(); + + std::fs::write(&app_o_file, &bytes).expect("failed to write object to file"); + + CodeGenTiming::default() +} + +fn gen_from_mono_module_dev_assembly( + arena: &bumpalo::Bump, + loaded: MonomorphizedModule, + target: &target_lexicon::Triple, + app_o_file: &Path, +) -> CodeGenTiming { + let lazy_literals = true; + let generate_allocators = false; // provided by the platform + + let env = roc_gen_dev::Env { + arena, + interns: loaded.interns, + exposed_to_host: loaded.exposed_to_host.keys().copied().collect(), + lazy_literals, + generate_allocators, + }; + + let module_object = roc_gen_dev::build_module(&env, target, loaded.procedures) + .expect("failed to compile module"); + + let module_out = module_object + .write() + .expect("failed to build output object"); + std::fs::write(&app_o_file, module_out).expect("failed to write object to file"); + + CodeGenTiming::default() +} diff --git a/compiler/build/src/target.rs b/compiler/build/src/target.rs index ba4f9c97b1..be13fc5d0f 100644 --- a/compiler/build/src/target.rs +++ b/compiler/build/src/target.rs @@ -106,7 +106,7 @@ pub fn target_machine( #[cfg(feature = "llvm")] pub fn convert_opt_level(level: OptLevel) -> OptimizationLevel { match level { - OptLevel::Normal => OptimizationLevel::None, + OptLevel::Development | OptLevel::Normal => OptimizationLevel::None, OptLevel::Optimize => OptimizationLevel::Aggressive, } } diff --git a/compiler/builtins/bitcode/src/main.zig b/compiler/builtins/bitcode/src/main.zig index 69bf1ca5ac..0841a8e8f5 100644 --- a/compiler/builtins/bitcode/src/main.zig +++ b/compiler/builtins/bitcode/src/main.zig @@ -101,6 +101,7 @@ comptime { exportStrFn(str.strToUtf8C, "to_utf8"); exportStrFn(str.fromUtf8C, "from_utf8"); exportStrFn(str.fromUtf8RangeC, "from_utf8_range"); + exportStrFn(str.repeat, "repeat"); } // Utils @@ -164,7 +165,12 @@ test "" { // https://github.com/ziglang/zig/blob/85755c51d529e7d9b406c6bdf69ce0a0f33f3353/lib/std/special/compiler_rt/muloti4.zig // // Thank you Zig Contributors! -export fn __muloti4(a: i128, b: i128, overflow: *c_int) callconv(.C) i128 { + +// Export it as weak incase it is alreadly linked in by something else. +comptime { + @export(__muloti4, .{ .name = "__muloti4", .linkage = .Weak }); +} +fn __muloti4(a: i128, b: i128, overflow: *c_int) callconv(.C) i128 { // @setRuntimeSafety(std.builtin.is_test); const min = @bitCast(i128, @as(u128, 1 << (128 - 1))); diff --git a/compiler/builtins/bitcode/src/str.zig b/compiler/builtins/bitcode/src/str.zig index 801550663c..ac2b3f6399 100644 --- a/compiler/builtins/bitcode/src/str.zig +++ b/compiler/builtins/bitcode/src/str.zig @@ -866,6 +866,22 @@ pub fn startsWith(string: RocStr, prefix: RocStr) callconv(.C) bool { return true; } +// Str.repeat +pub fn repeat(string: RocStr, count: usize) callconv(.C) RocStr { + const bytes_len = string.len(); + const bytes_ptr = string.asU8ptr(); + + var ret_string = RocStr.allocate(.Clone, count * bytes_len); + var ret_string_ptr = ret_string.asU8ptr(); + + var i: usize = 0; + while (i < count) : (i += 1) { + @memcpy(ret_string_ptr + (i * bytes_len), bytes_ptr, bytes_len); + } + + return ret_string; +} + // Str.startsWithCodePt pub fn startsWithCodePt(string: RocStr, prefix: u32) callconv(.C) bool { const bytes_ptr = string.asU8ptr(); @@ -1150,8 +1166,8 @@ fn strToBytes(arg: RocStr) RocList { } const FromUtf8Result = extern struct { - string: RocStr, byte_index: usize, + string: RocStr, is_ok: bool, problem_code: Utf8ByteProblem, }; diff --git a/compiler/builtins/src/bitcode.rs b/compiler/builtins/src/bitcode.rs index b45d822c69..fd1ac7b17c 100644 --- a/compiler/builtins/src/bitcode.rs +++ b/compiler/builtins/src/bitcode.rs @@ -28,6 +28,7 @@ pub const STR_EQUAL: &str = "roc_builtins.str.equal"; pub const STR_TO_UTF8: &str = "roc_builtins.str.to_utf8"; pub const STR_FROM_UTF8: &str = "roc_builtins.str.from_utf8"; pub const STR_FROM_UTF8_RANGE: &str = "roc_builtins.str.from_utf8_range"; +pub const STR_REPEAT: &str = "roc_builtins.str.repeat"; pub const DICT_HASH: &str = "roc_builtins.dict.hash"; pub const DICT_HASH_STR: &str = "roc_builtins.dict.hash_str"; diff --git a/compiler/builtins/src/std.rs b/compiler/builtins/src/std.rs index 98762b568b..7615879227 100644 --- a/compiler/builtins/src/std.rs +++ b/compiler/builtins/src/std.rs @@ -618,6 +618,13 @@ pub fn types() -> MutMap { Box::new(str_type()) ); + // repeat : Str, Nat -> Str + add_top_level_function_type!( + Symbol::STR_REPEAT, + vec![str_type(), nat_type()], + Box::new(str_type()) + ); + // fromUtf8 : List U8 -> Result Str [ BadUtf8 Utf8Problem ]* { let bad_utf8 = SolvedType::TagUnion( diff --git a/compiler/can/src/annotation.rs b/compiler/can/src/annotation.rs index 5bade7aeee..bbb1707ef2 100644 --- a/compiler/can/src/annotation.rs +++ b/compiler/can/src/annotation.rs @@ -82,6 +82,7 @@ pub fn canonicalize_annotation( let mut introduced_variables = IntroducedVariables::default(); let mut references = MutSet::default(); let mut aliases = SendMap::default(); + let typ = can_annotation_help( env, annotation, @@ -249,28 +250,7 @@ fn can_annotation_help( actual: Box::new(actual), } } - None => { - let mut args = Vec::new(); - - references.insert(symbol); - - for arg in *type_arguments { - let arg_ann = can_annotation_help( - env, - &arg.value, - region, - scope, - var_store, - introduced_variables, - local_aliases, - references, - ); - - args.push(arg_ann); - } - - Type::Apply(symbol, args) - } + None => Type::Apply(symbol, args), } } BoundVariable(v) => { diff --git a/compiler/can/src/builtins.rs b/compiler/can/src/builtins.rs index fe45f1f673..d4fff2ef74 100644 --- a/compiler/can/src/builtins.rs +++ b/compiler/can/src/builtins.rs @@ -66,6 +66,7 @@ pub fn builtin_defs_map(symbol: Symbol, var_store: &mut VarStore) -> Option STR_FROM_UTF8_RANGE => str_from_utf8_range, STR_TO_UTF8 => str_to_utf8, STR_FROM_FLOAT=> str_from_float, + STR_REPEAT => str_repeat, LIST_LEN => list_len, LIST_GET => list_get, LIST_SET => list_set, @@ -1233,6 +1234,26 @@ fn str_split(symbol: Symbol, var_store: &mut VarStore) -> Def { ) } +/// Str.repeat : Str, Nat -> Str +fn str_repeat(symbol: Symbol, var_store: &mut VarStore) -> Def { + let str_var = var_store.fresh(); + let nat_var = var_store.fresh(); + + let body = RunLowLevel { + op: LowLevel::StrRepeat, + args: vec![(str_var, Var(Symbol::ARG_1)), (nat_var, Var(Symbol::ARG_2))], + ret_var: str_var, + }; + + defn( + symbol, + vec![(str_var, Symbol::ARG_1), (nat_var, Symbol::ARG_2)], + var_store, + body, + str_var, + ) +} + /// Str.concat : Str, Str -> Str fn str_concat(symbol: Symbol, var_store: &mut VarStore) -> Def { let str_var = var_store.fresh(); diff --git a/compiler/gen_dev/src/generic64/aarch64.rs b/compiler/gen_dev/src/generic64/aarch64.rs index 6149bff14c..32273fbf91 100644 --- a/compiler/gen_dev/src/generic64/aarch64.rs +++ b/compiler/gen_dev/src/generic64/aarch64.rs @@ -226,6 +226,7 @@ impl CallConv for AArch64Call { #[inline(always)] fn load_args<'a>( + _buf: &mut Vec<'a, u8>, _symbol_map: &mut MutMap>, _args: &'a [(Layout<'a>, Symbol)], _ret_layout: &Layout<'a>, diff --git a/compiler/gen_dev/src/generic64/mod.rs b/compiler/gen_dev/src/generic64/mod.rs index 171a92ca9a..008633c202 100644 --- a/compiler/gen_dev/src/generic64/mod.rs +++ b/compiler/gen_dev/src/generic64/mod.rs @@ -2,15 +2,14 @@ use crate::{Backend, Env, Relocation}; use bumpalo::collections::Vec; use roc_collections::all::{MutMap, MutSet}; use roc_module::symbol::Symbol; -use roc_mono::ir::{BranchInfo, Literal, Stmt}; +use roc_mono::ir::{BranchInfo, JoinPointId, Literal, Param, SelfRecursive, Stmt}; use roc_mono::layout::{Builtin, Layout}; use std::marker::PhantomData; -use target_lexicon::Triple; pub mod aarch64; pub mod x86_64; -const PTR_SIZE: u32 = 64; +const PTR_SIZE: u32 = 8; pub trait CallConv { const GENERAL_PARAM_REGS: &'static [GeneralReg]; @@ -49,6 +48,7 @@ pub trait CallConv { // load_args updates the symbol map to know where every arg is stored. fn load_args<'a>( + buf: &mut Vec<'a, u8>, symbol_map: &mut MutMap>, args: &'a [(Layout<'a>, Symbol)], // ret_layout is needed because if it is a complex type, we pass a pointer as the first arg. @@ -211,12 +211,16 @@ pub struct Backend64Bit< env: &'a Env<'a>, buf: Vec<'a, u8>, relocs: Vec<'a, Relocation>, + proc_name: Option, + is_self_recursive: Option, last_seen_map: MutMap>, - layout_map: MutMap>, + layout_map: MutMap>, free_map: MutMap<*const Stmt<'a>, Vec<'a, Symbol>>, + symbol_storage_map: MutMap>, literal_map: MutMap>, + join_map: MutMap, // This should probably be smarter than a vec. // There are certain registers we should always use first. With pushing and popping, this could get mixed. @@ -247,11 +251,13 @@ impl< CC: CallConv, > Backend<'a> for Backend64Bit<'a, GeneralReg, FloatReg, ASM, CC> { - fn new(env: &'a Env, _target: &Triple) -> Result { + fn new(env: &'a Env) -> Result { Ok(Backend64Bit { phantom_asm: PhantomData, phantom_cc: PhantomData, env, + proc_name: None, + is_self_recursive: None, buf: bumpalo::vec![in env.arena], relocs: bumpalo::vec![in env.arena], last_seen_map: MutMap::default(), @@ -259,6 +265,7 @@ impl< free_map: MutMap::default(), symbol_storage_map: MutMap::default(), literal_map: MutMap::default(), + join_map: MutMap::default(), general_free_regs: bumpalo::vec![in env.arena], general_used_regs: bumpalo::vec![in env.arena], general_used_callee_saved_regs: MutSet::default(), @@ -275,12 +282,15 @@ impl< self.env } - fn reset(&mut self) { + fn reset(&mut self, name: String, is_self_recursive: SelfRecursive) { + self.proc_name = Some(name); + self.is_self_recursive = Some(is_self_recursive); self.stack_size = 0; self.free_stack_chunks.clear(); self.fn_call_stack_size = 0; self.last_seen_map.clear(); self.layout_map.clear(); + self.join_map.clear(); self.free_map.clear(); self.symbol_storage_map.clear(); self.buf.clear(); @@ -304,7 +314,7 @@ impl< &mut self.last_seen_map } - fn layout_map(&mut self) -> &mut MutMap> { + fn layout_map(&mut self) -> &mut MutMap> { &mut self.layout_map } @@ -330,8 +340,49 @@ impl< )?; let setup_offset = out.len(); + // Deal with jumps to the return address. + let old_relocs = std::mem::replace(&mut self.relocs, bumpalo::vec![in self.env.arena]); + + // Check if their is an unnessary jump to return right at the end of the function. + let mut end_jmp_size = 0; + for reloc in old_relocs + .iter() + .filter(|reloc| matches!(reloc, Relocation::JmpToReturn { .. })) + { + if let Relocation::JmpToReturn { + inst_loc, + inst_size, + .. + } = reloc + { + if *inst_loc as usize + *inst_size as usize == self.buf.len() { + end_jmp_size = *inst_size as usize; + break; + } + } + } + + // Update jumps to returns. + let ret_offset = self.buf.len() - end_jmp_size; + let mut tmp = bumpalo::vec![in self.env.arena]; + for reloc in old_relocs + .iter() + .filter(|reloc| matches!(reloc, Relocation::JmpToReturn { .. })) + { + if let Relocation::JmpToReturn { + inst_loc, + inst_size, + offset, + } = reloc + { + if *inst_loc as usize + *inst_size as usize != self.buf.len() { + self.update_jmp_imm32_offset(&mut tmp, *inst_loc, *offset, ret_offset as u64); + } + } + } + // Add function body. - out.extend(&self.buf); + out.extend(&self.buf[..self.buf.len() - end_jmp_size]); // Cleanup stack. CC::cleanup_stack( @@ -342,23 +393,28 @@ impl< )?; ASM::ret(&mut out); - // Update relocs to include stack setup offset. + // Update other relocs to include stack setup offset. let mut out_relocs = bumpalo::vec![in self.env.arena]; - let old_relocs = std::mem::replace(&mut self.relocs, bumpalo::vec![in self.env.arena]); - out_relocs.extend(old_relocs.into_iter().map(|reloc| match reloc { - Relocation::LocalData { offset, data } => Relocation::LocalData { - offset: offset + setup_offset as u64, - data, - }, - Relocation::LinkedData { offset, name } => Relocation::LinkedData { - offset: offset + setup_offset as u64, - name, - }, - Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { - offset: offset + setup_offset as u64, - name, - }, - })); + out_relocs.extend( + old_relocs + .into_iter() + .filter(|reloc| !matches!(reloc, Relocation::JmpToReturn { .. })) + .map(|reloc| match reloc { + Relocation::LocalData { offset, data } => Relocation::LocalData { + offset: offset + setup_offset as u64, + data, + }, + Relocation::LinkedData { offset, name } => Relocation::LinkedData { + offset: offset + setup_offset as u64, + name, + }, + Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { + offset: offset + setup_offset as u64, + name, + }, + Relocation::JmpToReturn { .. } => unreachable!(), + }), + ); Ok((out.into_bump_slice(), out_relocs.into_bump_slice())) } @@ -367,7 +423,12 @@ impl< args: &'a [(Layout<'a>, Symbol)], ret_layout: &Layout<'a>, ) -> Result<(), String> { - CC::load_args(&mut self.symbol_storage_map, args, ret_layout)?; + CC::load_args( + &mut self.buf, + &mut self.symbol_storage_map, + args, + ret_layout, + )?; // Update used and free regs. for (sym, storage) in &self.symbol_storage_map { match storage { @@ -401,29 +462,13 @@ impl< arg_layouts: &[Layout<'a>], ret_layout: &Layout<'a>, ) -> Result<(), String> { + if let Some(SelfRecursive::SelfRecursive(id)) = self.is_self_recursive { + if &fn_name == self.proc_name.as_ref().unwrap() && self.join_map.contains_key(&id) { + return self.build_jump(&id, args, arg_layouts, ret_layout); + } + } // Save used caller saved regs. - let old_general_used_regs = std::mem::replace( - &mut self.general_used_regs, - bumpalo::vec![in self.env.arena], - ); - for (reg, saved_sym) in old_general_used_regs.into_iter() { - if CC::general_caller_saved(®) { - self.general_free_regs.push(reg); - self.free_to_stack(&saved_sym)?; - } else { - self.general_used_regs.push((reg, saved_sym)); - } - } - let old_float_used_regs = - std::mem::replace(&mut self.float_used_regs, bumpalo::vec![in self.env.arena]); - for (reg, saved_sym) in old_float_used_regs.into_iter() { - if CC::float_caller_saved(®) { - self.float_free_regs.push(reg); - self.free_to_stack(&saved_sym)?; - } else { - self.float_used_regs.push((reg, saved_sym)); - } - } + self.push_used_caller_saved_regs_to_stack()?; // Put values in param regs or on top of the stack. let tmp_stack_size = CC::store_args( @@ -450,6 +495,25 @@ impl< ASM::mov_freg64_freg64(&mut self.buf, dst_reg, CC::FLOAT_RETURN_REGS[0]); Ok(()) } + Layout::Builtin(Builtin::Str) => { + if CC::returns_via_arg_pointer(ret_layout)? { + // This will happen on windows, return via pointer here. + Err("FnCall: Returning strings via pointer not yet implemented".to_string()) + } else { + let offset = self.claim_stack_size(16)?; + self.symbol_storage_map.insert( + *dst, + SymbolStorage::Base { + offset, + size: 16, + owned: true, + }, + ); + ASM::mov_base32_reg64(&mut self.buf, offset, CC::GENERAL_RETURN_REGS[0]); + ASM::mov_base32_reg64(&mut self.buf, offset + 8, CC::GENERAL_RETURN_REGS[1]); + Ok(()) + } + } x => Err(format!( "FnCall: receiving return type, {:?}, is not yet implemented", x @@ -486,7 +550,7 @@ impl< // Build unconditional jump to the end of this switch. // Since we don't know the offset yet, set it to 0 and overwrite later. let jmp_location = self.buf.len(); - let jmp_offset = ASM::jmp_imm32(&mut self.buf, 0); + let jmp_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); ret_jumps.push((jmp_location, jmp_offset)); // Overwite the original jne with the correct offset. @@ -510,12 +574,12 @@ impl< // Update all return jumps to jump past the default case. let ret_offset = self.buf.len(); for (jmp_location, start_offset) in ret_jumps.into_iter() { - tmp.clear(); - let jmp_offset = ret_offset - start_offset; - ASM::jmp_imm32(&mut tmp, jmp_offset as i32); - for (i, byte) in tmp.iter().enumerate() { - self.buf[jmp_location + i] = *byte; - } + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + ret_offset as u64, + ); } Ok(()) } else { @@ -526,6 +590,134 @@ impl< } } + fn build_join( + &mut self, + id: &JoinPointId, + parameters: &'a [Param<'a>], + body: &'a Stmt<'a>, + remainder: &'a Stmt<'a>, + ret_layout: &Layout<'a>, + ) -> Result<(), String> { + // Create jump to remaining. + let jmp_location = self.buf.len(); + let start_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); + + // This section can essentially be seen as a sub function within the main function. + // Thus we build using a new backend with some minor extra synchronization. + let mut sub_backend = Self::new(self.env)?; + sub_backend.reset( + self.proc_name.as_ref().unwrap().clone(), + self.is_self_recursive.as_ref().unwrap().clone(), + ); + // Sync static maps of important information. + sub_backend.last_seen_map = self.last_seen_map.clone(); + sub_backend.layout_map = self.layout_map.clone(); + sub_backend.free_map = self.free_map.clone(); + + // Setup join point. + sub_backend.join_map.insert(*id, 0); + self.join_map.insert(*id, self.buf.len() as u64); + + // Sync stack size so the "sub function" doesn't mess up our stack. + sub_backend.stack_size = self.stack_size; + sub_backend.fn_call_stack_size = self.fn_call_stack_size; + + // Load params as if they were args. + let mut args = bumpalo::vec![in self.env.arena]; + for param in parameters { + args.push((param.layout, param.symbol)); + } + sub_backend.load_args(args.into_bump_slice(), ret_layout)?; + + // Build all statements in body. + sub_backend.build_stmt(body, ret_layout)?; + + // Merge the "sub function" into the main function. + let sub_func_offset = self.buf.len() as u64; + self.buf.extend_from_slice(&sub_backend.buf); + // Update stack based on how much was used by the sub function. + self.stack_size = sub_backend.stack_size; + self.fn_call_stack_size = sub_backend.fn_call_stack_size; + // Relocations must be shifted to be merged correctly. + self.relocs + .extend(sub_backend.relocs.into_iter().map(|reloc| match reloc { + Relocation::LocalData { offset, data } => Relocation::LocalData { + offset: offset + sub_func_offset, + data, + }, + Relocation::LinkedData { offset, name } => Relocation::LinkedData { + offset: offset + sub_func_offset, + name, + }, + Relocation::LinkedFunction { offset, name } => Relocation::LinkedFunction { + offset: offset + sub_func_offset, + name, + }, + Relocation::JmpToReturn { + inst_loc, + inst_size, + offset, + } => Relocation::JmpToReturn { + inst_loc: inst_loc + sub_func_offset, + inst_size, + offset: offset + sub_func_offset, + }, + })); + + // Overwite the original jump with the correct offset. + let mut tmp = bumpalo::vec![in self.env.arena]; + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + self.buf.len() as u64, + ); + + // Build remainder of function. + self.build_stmt(remainder, ret_layout) + } + + fn build_jump( + &mut self, + id: &JoinPointId, + args: &'a [Symbol], + arg_layouts: &[Layout<'a>], + ret_layout: &Layout<'a>, + ) -> Result<(), String> { + // Treat this like a function call, but with a jump instead of a call instruction at the end. + + self.push_used_caller_saved_regs_to_stack()?; + + let tmp_stack_size = CC::store_args( + &mut self.buf, + &self.symbol_storage_map, + args, + arg_layouts, + ret_layout, + )?; + self.fn_call_stack_size = std::cmp::max(self.fn_call_stack_size, tmp_stack_size); + + let jmp_location = self.buf.len(); + let start_offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678); + + if let Some(offset) = self.join_map.get(id) { + let offset = *offset; + let mut tmp = bumpalo::vec![in self.env.arena]; + self.update_jmp_imm32_offset( + &mut tmp, + jmp_location as u64, + start_offset as u64, + offset, + ); + Ok(()) + } else { + Err(format!( + "Jump: unknown point specified to jump to: {:?}", + id + )) + } + } + fn build_num_abs( &mut self, dst: &Symbol, @@ -726,6 +918,35 @@ impl< ASM::mov_freg64_imm64(&mut self.buf, &mut self.relocs, reg, val); Ok(()) } + Literal::Str(x) if x.len() < 16 => { + // Load small string. + let reg = self.get_tmp_general_reg()?; + + let offset = self.claim_stack_size(16)?; + self.symbol_storage_map.insert( + *sym, + SymbolStorage::Base { + offset, + size: 16, + owned: true, + }, + ); + let mut bytes = [0; 16]; + bytes[..x.len()].copy_from_slice(x.as_bytes()); + bytes[15] = (x.len() as u8) | 0b1000_0000; + + let mut num_bytes = [0; 8]; + num_bytes.copy_from_slice(&bytes[..8]); + let num = i64::from_ne_bytes(num_bytes); + ASM::mov_reg64_imm64(&mut self.buf, reg, num); + ASM::mov_base32_reg64(&mut self.buf, offset, reg); + + num_bytes.copy_from_slice(&bytes[8..]); + let num = i64::from_ne_bytes(num_bytes); + ASM::mov_reg64_imm64(&mut self.buf, reg, num); + ASM::mov_base32_reg64(&mut self.buf, offset + 8, reg); + Ok(()) + } x => Err(format!("loading literal, {:?}, is not yet implemented", x)), } } @@ -828,29 +1049,39 @@ impl< fn return_symbol(&mut self, sym: &Symbol, layout: &Layout<'a>) -> Result<(), String> { let val = self.symbol_storage_map.get(sym); match val { - Some(SymbolStorage::GeneralReg(reg)) if *reg == CC::GENERAL_RETURN_REGS[0] => Ok(()), + Some(SymbolStorage::GeneralReg(reg)) if *reg == CC::GENERAL_RETURN_REGS[0] => {} Some(SymbolStorage::GeneralReg(reg)) => { // If it fits in a general purpose register, just copy it over to. // Technically this can be optimized to produce shorter instructions if less than 64bits. ASM::mov_reg64_reg64(&mut self.buf, CC::GENERAL_RETURN_REGS[0], *reg); - Ok(()) } - Some(SymbolStorage::FloatReg(reg)) if *reg == CC::FLOAT_RETURN_REGS[0] => Ok(()), + Some(SymbolStorage::FloatReg(reg)) if *reg == CC::FLOAT_RETURN_REGS[0] => {} Some(SymbolStorage::FloatReg(reg)) => { ASM::mov_freg64_freg64(&mut self.buf, CC::FLOAT_RETURN_REGS[0], *reg); - Ok(()) } Some(SymbolStorage::Base { offset, size, .. }) => match layout { Layout::Builtin(Builtin::Int64) => { ASM::mov_reg64_base32(&mut self.buf, CC::GENERAL_RETURN_REGS[0], *offset); - Ok(()) } Layout::Builtin(Builtin::Float64) => { ASM::mov_freg64_base32(&mut self.buf, CC::FLOAT_RETURN_REGS[0], *offset); - Ok(()) + } + Layout::Builtin(Builtin::Str) => { + if self.symbol_storage_map.contains_key(&Symbol::RET_POINTER) { + // This will happen on windows, return via pointer here. + return Err("Returning strings via pointer not yet implemented".to_string()); + } else { + ASM::mov_reg64_base32(&mut self.buf, CC::GENERAL_RETURN_REGS[0], *offset); + ASM::mov_reg64_base32( + &mut self.buf, + CC::GENERAL_RETURN_REGS[1], + *offset + 8, + ); + } } Layout::Struct(field_layouts) => { let (offset, size) = (*offset, *size); + // Nothing to do for empty struct if size > 0 { let ret_reg = if self.symbol_storage_map.contains_key(&Symbol::RET_POINTER) { @@ -858,23 +1089,34 @@ impl< } else { None }; - CC::return_struct(&mut self.buf, offset, size, field_layouts, ret_reg) - } else { - // Nothing to do for empty struct - Ok(()) + CC::return_struct(&mut self.buf, offset, size, field_layouts, ret_reg)?; } } - x => Err(format!( - "returning symbol with layout, {:?}, is not yet implemented", - x - )), + x => { + return Err(format!( + "returning symbol with layout, {:?}, is not yet implemented", + x + )); + } }, - Some(x) => Err(format!( - "returning symbol storage, {:?}, is not yet implemented", - x - )), - None => Err(format!("Unknown return symbol: {}", sym)), + Some(x) => { + return Err(format!( + "returning symbol storage, {:?}, is not yet implemented", + x + )); + } + None => { + return Err(format!("Unknown return symbol: {}", sym)); + } } + let inst_loc = self.buf.len() as u64; + let offset = ASM::jmp_imm32(&mut self.buf, 0x1234_5678) as u64; + self.relocs.push(Relocation::JmpToReturn { + inst_loc, + inst_size: self.buf.len() as u64 - inst_loc, + offset, + }); + Ok(()) } } @@ -1212,4 +1454,72 @@ impl< )), } } + + fn push_used_caller_saved_regs_to_stack(&mut self) -> Result<(), String> { + let old_general_used_regs = std::mem::replace( + &mut self.general_used_regs, + bumpalo::vec![in self.env.arena], + ); + for (reg, saved_sym) in old_general_used_regs.into_iter() { + if CC::general_caller_saved(®) { + self.general_free_regs.push(reg); + self.free_to_stack(&saved_sym)?; + } else { + self.general_used_regs.push((reg, saved_sym)); + } + } + let old_float_used_regs = + std::mem::replace(&mut self.float_used_regs, bumpalo::vec![in self.env.arena]); + for (reg, saved_sym) in old_float_used_regs.into_iter() { + if CC::float_caller_saved(®) { + self.float_free_regs.push(reg); + self.free_to_stack(&saved_sym)?; + } else { + self.float_used_regs.push((reg, saved_sym)); + } + } + Ok(()) + } + + // Updates a jump instruction to a new offset and returns the number of bytes written. + fn update_jmp_imm32_offset( + &mut self, + tmp: &mut Vec<'a, u8>, + jmp_location: u64, + base_offset: u64, + target_offset: u64, + ) { + tmp.clear(); + let jmp_offset = target_offset as i32 - base_offset as i32; + ASM::jmp_imm32(tmp, jmp_offset); + for (i, byte) in tmp.iter().enumerate() { + self.buf[jmp_location as usize + i] = *byte; + } + } +} + +#[macro_export] +macro_rules! single_register_integers { + () => { + Builtin::Int1 + | Builtin::Int8 + | Builtin::Int16 + | Builtin::Int32 + | Builtin::Int64 + | Builtin::Usize + }; +} + +#[macro_export] +macro_rules! single_register_floats { + () => { + Builtin::Float32 | Builtin::Float64 + }; +} + +#[macro_export] +macro_rules! single_register_builtins { + () => { + single_register_integers!() | single_register_floats!() + }; } diff --git a/compiler/gen_dev/src/generic64/x86_64.rs b/compiler/gen_dev/src/generic64/x86_64.rs index d052c44e43..a30a1a4e9c 100644 --- a/compiler/gen_dev/src/generic64/x86_64.rs +++ b/compiler/gen_dev/src/generic64/x86_64.rs @@ -1,5 +1,7 @@ use crate::generic64::{Assembler, CallConv, RegTrait, SymbolStorage, PTR_SIZE}; -use crate::Relocation; +use crate::{ + single_register_builtins, single_register_floats, single_register_integers, Relocation, +}; use bumpalo::collections::Vec; use roc_collections::all::MutMap; use roc_module::symbol::Symbol; @@ -175,6 +177,7 @@ impl CallConv for X86_64SystemV { #[inline(always)] fn load_args<'a>( + buf: &mut Vec<'a, u8>, symbol_map: &mut MutMap>, args: &'a [(Layout<'a>, Symbol)], ret_layout: &Layout<'a>, @@ -191,7 +194,7 @@ impl CallConv for X86_64SystemV { } for (layout, sym) in args.iter() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if general_i < Self::GENERAL_PARAM_REGS.len() { symbol_map.insert( *sym, @@ -210,7 +213,7 @@ impl CallConv for X86_64SystemV { ); } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if float_i < Self::FLOAT_PARAM_REGS.len() { symbol_map.insert( *sym, @@ -229,6 +232,30 @@ impl CallConv for X86_64SystemV { ); } } + Layout::Builtin(Builtin::Str) => { + if general_i + 1 < Self::GENERAL_PARAM_REGS.len() { + // Load the value to the param reg. + let dst1 = Self::GENERAL_PARAM_REGS[general_i]; + let dst2 = Self::GENERAL_PARAM_REGS[general_i + 1]; + base_offset += 16; + X86_64Assembler::mov_reg64_base32(buf, dst1, base_offset - 8); + X86_64Assembler::mov_reg64_base32(buf, dst2, base_offset); + symbol_map.insert( + *sym, + SymbolStorage::Base { + offset: base_offset, + size: 16, + owned: true, + }, + ); + general_i += 2; + } else { + return Err( + "loading strings args on the stack is not yet implemented".to_string() + ); + } + } + Layout::Struct(&[]) => {} x => { return Err(format!( "Loading args with layout {:?} not yet implementd", @@ -254,8 +281,7 @@ impl CallConv for X86_64SystemV { // For most return layouts we will do nothing. // In some cases, we need to put the return address as the first arg. match ret_layout { - Layout::Builtin(Builtin::Int64) => {} - Layout::Builtin(Builtin::Float64) => {} + Layout::Builtin(single_register_builtins!() | Builtin::Str) => {} x => { return Err(format!( "receiving return type, {:?}, is not yet implemented", @@ -265,7 +291,7 @@ impl CallConv for X86_64SystemV { } for (i, layout) in arg_layouts.iter().enumerate() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if general_i < Self::GENERAL_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::GENERAL_PARAM_REGS[general_i]; @@ -319,7 +345,7 @@ impl CallConv for X86_64SystemV { stack_offset += 8; } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if float_i < Self::FLOAT_PARAM_REGS.len() { // Load the value to the param reg. let dst = Self::FLOAT_PARAM_REGS[float_i]; @@ -371,6 +397,33 @@ impl CallConv for X86_64SystemV { stack_offset += 8; } } + Layout::Builtin(Builtin::Str) => { + if general_i + 1 < Self::GENERAL_PARAM_REGS.len() { + // Load the value to the param reg. + let dst1 = Self::GENERAL_PARAM_REGS[general_i]; + let dst2 = Self::GENERAL_PARAM_REGS[general_i + 1]; + match symbol_map + .get(&args[i]) + .ok_or("function argument does not reference any symbol")? + { + SymbolStorage::Base { offset, .. } => { + X86_64Assembler::mov_reg64_base32(buf, dst1, *offset); + X86_64Assembler::mov_reg64_base32(buf, dst2, *offset + 8); + } + _ => { + return Err("Strings only support being loaded from base offsets" + .to_string()); + } + } + general_i += 2; + } else { + return Err( + "calling functions with strings on the stack is not yet implemented" + .to_string(), + ); + } + } + Layout::Struct(&[]) => {} x => { return Err(format!( "calling with arg type, {:?}, is not yet implemented", @@ -513,6 +566,7 @@ impl CallConv for X86_64WindowsFastcall { #[inline(always)] fn load_args<'a>( + _buf: &mut Vec<'a, u8>, symbol_map: &mut MutMap>, args: &'a [(Layout<'a>, Symbol)], ret_layout: &Layout<'a>, @@ -529,13 +583,23 @@ impl CallConv for X86_64WindowsFastcall { for (layout, sym) in args.iter() { if i < Self::GENERAL_PARAM_REGS.len() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { symbol_map .insert(*sym, SymbolStorage::GeneralReg(Self::GENERAL_PARAM_REGS[i])); + i += 1; } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { symbol_map.insert(*sym, SymbolStorage::FloatReg(Self::FLOAT_PARAM_REGS[i])); + i += 1; } + Layout::Builtin(Builtin::Str) => { + // I think this just needs to be passed on the stack, so not a huge deal. + return Err( + "Passing str args with Windows fast call not yet implemented." + .to_string(), + ); + } + Layout::Struct(&[]) => {} x => { return Err(format!( "Loading args with layout {:?} not yet implementd", @@ -543,11 +607,9 @@ impl CallConv for X86_64WindowsFastcall { )); } } - i += 1; } else { base_offset += match layout { - Layout::Builtin(Builtin::Int64) => 8, - Layout::Builtin(Builtin::Float64) => 8, + Layout::Builtin(single_register_builtins!()) => 8, x => { return Err(format!( "Loading args with layout {:?} not yet implemented", @@ -577,12 +639,10 @@ impl CallConv for X86_64WindowsFastcall { ret_layout: &Layout<'a>, ) -> Result { let mut stack_offset = Self::SHADOW_SPACE_SIZE as i32; - let mut reg_i = 0; // For most return layouts we will do nothing. // In some cases, we need to put the return address as the first arg. match ret_layout { - Layout::Builtin(Builtin::Int64) => {} - Layout::Builtin(Builtin::Float64) => {} + Layout::Builtin(single_register_builtins!()) => {} x => { return Err(format!( "receiving return type, {:?}, is not yet implemented", @@ -592,10 +652,10 @@ impl CallConv for X86_64WindowsFastcall { } for (i, layout) in arg_layouts.iter().enumerate() { match layout { - Layout::Builtin(Builtin::Int64) => { + Layout::Builtin(single_register_integers!()) => { if i < Self::GENERAL_PARAM_REGS.len() { // Load the value to the param reg. - let dst = Self::GENERAL_PARAM_REGS[reg_i]; + let dst = Self::GENERAL_PARAM_REGS[i]; match symbol_map .get(&args[i]) .ok_or("function argument does not reference any symbol")? @@ -613,7 +673,6 @@ impl CallConv for X86_64WindowsFastcall { ) } } - reg_i += 1; } else { // Load the value to the stack. match symbol_map @@ -646,10 +705,10 @@ impl CallConv for X86_64WindowsFastcall { stack_offset += 8; } } - Layout::Builtin(Builtin::Float64) => { + Layout::Builtin(single_register_floats!()) => { if i < Self::FLOAT_PARAM_REGS.len() { // Load the value to the param reg. - let dst = Self::FLOAT_PARAM_REGS[reg_i]; + let dst = Self::FLOAT_PARAM_REGS[i]; match symbol_map .get(&args[i]) .ok_or("function argument does not reference any symbol")? @@ -666,7 +725,6 @@ impl CallConv for X86_64WindowsFastcall { return Err("Cannot load general symbol into FloatReg".to_string()) } } - reg_i += 1; } else { // Load the value to the stack. match symbol_map @@ -698,6 +756,13 @@ impl CallConv for X86_64WindowsFastcall { stack_offset += 8; } } + Layout::Builtin(Builtin::Str) => { + // I think this just needs to be passed on the stack, so not a huge deal. + return Err( + "Passing str args with Windows fast call not yet implemented.".to_string(), + ); + } + Layout::Struct(&[]) => {} x => { return Err(format!( "calling with arg type, {:?}, is not yet implemented", diff --git a/compiler/gen_dev/src/lib.rs b/compiler/gen_dev/src/lib.rs index 70e9bf0f61..5ef03a3a42 100644 --- a/compiler/gen_dev/src/lib.rs +++ b/compiler/gen_dev/src/lib.rs @@ -9,10 +9,10 @@ use roc_module::ident::{ModuleName, TagName}; use roc_module::low_level::LowLevel; use roc_module::symbol::{Interns, Symbol}; use roc_mono::ir::{ - BranchInfo, CallType, Expr, JoinPointId, ListLiteralElement, Literal, Proc, Stmt, + BranchInfo, CallType, Expr, JoinPointId, ListLiteralElement, Literal, Param, Proc, + SelfRecursive, Stmt, }; use roc_mono::layout::{Builtin, Layout, LayoutIds}; -use target_lexicon::Triple; mod generic64; mod object_builder; @@ -46,6 +46,11 @@ pub enum Relocation { offset: u64, name: String, }, + JmpToReturn { + inst_loc: u64, + inst_size: u64, + offset: u64, + }, } trait Backend<'a> @@ -53,12 +58,13 @@ where Self: Sized, { /// new creates a new backend that will output to the specific Object. - fn new(env: &'a Env, target: &Triple) -> Result; + fn new(env: &'a Env) -> Result; fn env(&self) -> &'a Env<'a>; /// reset resets any registers or other values that may be occupied at the end of a procedure. - fn reset(&mut self); + /// It also passes basic procedure information to the builder for setup of the next function. + fn reset(&mut self, name: String, is_self_recursive: SelfRecursive); /// finalize does any setup and cleanup that should happen around the procedure. /// finalize does setup because things like stack size and jump locations are not know until the function is written. @@ -79,17 +85,16 @@ where /// build_proc creates a procedure and outputs it to the wrapped object writer. fn build_proc(&mut self, proc: Proc<'a>) -> Result<(&'a [u8], &[Relocation]), String> { - self.reset(); + let proc_name = LayoutIds::default() + .get(proc.name, &proc.ret_layout) + .to_symbol_string(proc.name, &self.env().interns); + self.reset(proc_name, proc.is_self_recursive); self.load_args(proc.args, &proc.ret_layout)?; for (layout, sym) in proc.args { self.set_layout_map(*sym, layout)?; } - // let start = std::time::Instant::now(); self.scan_ast(&proc.body); self.create_free_map(); - // let duration = start.elapsed(); - // println!("Time to calculate lifetimes: {:?}", duration); - // println!("{:?}", self.last_seen_map()); self.build_stmt(&proc.body, &proc.ret_layout)?; self.finalize() } @@ -110,6 +115,11 @@ where self.free_symbols(stmt)?; Ok(()) } + Stmt::Refcounting(_modify, following) => { + // TODO: actually deal with refcounting. For hello world, we just skipped it. + self.build_stmt(following, ret_layout)?; + Ok(()) + } Stmt::Switch { cond_symbol, cond_layout, @@ -128,6 +138,35 @@ where self.free_symbols(stmt)?; Ok(()) } + Stmt::Join { + id, + parameters, + body, + remainder, + } => { + for param in parameters.iter() { + self.set_layout_map(param.symbol, ¶m.layout)?; + } + self.build_join(id, parameters, body, remainder, ret_layout)?; + self.free_symbols(stmt)?; + Ok(()) + } + Stmt::Jump(id, args) => { + let mut arg_layouts: bumpalo::collections::Vec> = + bumpalo::vec![in self.env().arena]; + arg_layouts.reserve(args.len()); + let layout_map = self.layout_map(); + for arg in *args { + if let Some(layout) = layout_map.get(arg) { + arg_layouts.push(*layout); + } else { + return Err(format!("the argument, {:?}, has no know layout", arg)); + } + } + self.build_jump(id, args, arg_layouts.into_bump_slice(), ret_layout)?; + self.free_symbols(stmt)?; + Ok(()) + } x => Err(format!("the statement, {:?}, is not yet implemented", x)), } } @@ -141,6 +180,25 @@ where ret_layout: &Layout<'a>, ) -> Result<(), String>; + // build_join generates a instructions for a join statement. + fn build_join( + &mut self, + id: &JoinPointId, + parameters: &'a [Param<'a>], + body: &'a Stmt<'a>, + remainder: &'a Stmt<'a>, + ret_layout: &Layout<'a>, + ) -> Result<(), String>; + + // build_jump generates a instructions for a jump statement. + fn build_jump( + &mut self, + id: &JoinPointId, + args: &'a [Symbol], + arg_layouts: &[Layout<'a>], + ret_layout: &Layout<'a>, + ) -> Result<(), String>; + /// build_expr builds the expressions for the specified symbol. /// The builder must keep track of the symbol because it may be referred to later. fn build_expr( @@ -241,6 +299,13 @@ where arg_layouts, ret_layout, ), + Symbol::STR_CONCAT => self.build_run_low_level( + sym, + &LowLevel::StrConcat, + arguments, + arg_layouts, + ret_layout, + ), x if x .module_string(&self.env().interns) .starts_with(ModuleName::APP) => @@ -263,8 +328,7 @@ where let layout_map = self.layout_map(); for arg in *arguments { if let Some(layout) = layout_map.get(arg) { - // This is safe because every value in the map is always set with a valid layout and cannot be null. - arg_layouts.push(unsafe { *(*layout) }); + arg_layouts.push(*layout); } else { return Err(format!("the argument, {:?}, has no know layout", arg)); } @@ -414,6 +478,13 @@ where arg_layouts, ret_layout, ), + LowLevel::StrConcat => self.build_fn_call( + sym, + bitcode::STR_CONCAT.to_string(), + args, + arg_layouts, + ret_layout, + ), x => Err(format!("low level, {:?}. is not yet implemented", x)), } } @@ -507,7 +578,7 @@ where /// load_literal sets a symbol to be equal to a literal. fn load_literal(&mut self, sym: &Symbol, lit: &Literal<'a>) -> Result<(), String>; - /// return_symbol moves a symbol to the correct return location for the backend. + /// return_symbol moves a symbol to the correct return location for the backend and adds a jump to the end of the function. fn return_symbol(&mut self, sym: &Symbol, layout: &Layout<'a>) -> Result<(), String>; /// free_symbols will free all symbols for the given statement. @@ -542,12 +613,10 @@ where /// set_layout_map sets the layout for a specific symbol. fn set_layout_map(&mut self, sym: Symbol, layout: &Layout<'a>) -> Result<(), String> { - if let Some(x) = self.layout_map().insert(sym, layout) { + if let Some(old_layout) = self.layout_map().insert(sym, *layout) { // Layout map already contains the symbol. We should never need to overwrite. // If the layout is not the same, that is a bug. - // There is always an old layout value and this dereference is safe. - let old_layout = unsafe { *x }; - if old_layout != *layout { + if &old_layout != layout { Err(format!( "Overwriting layout for symbol, {:?}. This should never happen. got {:?}, want {:?}", sym, layout, old_layout @@ -561,7 +630,7 @@ where } /// layout_map gets the map from symbol to layout. - fn layout_map(&mut self) -> &mut MutMap>; + fn layout_map(&mut self) -> &mut MutMap>; fn create_free_map(&mut self) { let mut free_map = MutMap::default(); diff --git a/compiler/gen_dev/src/object_builder.rs b/compiler/gen_dev/src/object_builder.rs index 52ae6b4b08..9e5ae6f76e 100644 --- a/compiler/gen_dev/src/object_builder.rs +++ b/compiler/gen_dev/src/object_builder.rs @@ -34,7 +34,7 @@ pub fn build_module<'a>( x86_64::X86_64FloatReg, x86_64::X86_64Assembler, x86_64::X86_64SystemV, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -52,7 +52,7 @@ pub fn build_module<'a>( x86_64::X86_64FloatReg, x86_64::X86_64Assembler, x86_64::X86_64SystemV, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -74,7 +74,7 @@ pub fn build_module<'a>( aarch64::AArch64FloatReg, aarch64::AArch64Assembler, aarch64::AArch64Call, - > = Backend::new(env, target)?; + > = Backend::new(env)?; build_object( env, procedures, @@ -191,10 +191,16 @@ fn build_object<'a, B: Backend<'a>>( let mut layout_ids = roc_mono::layout::LayoutIds::default(); let mut procs = Vec::with_capacity_in(procedures.len(), env.arena); for ((sym, layout), proc) in procedures { - let fn_name = layout_ids + let base_name = layout_ids .get_toplevel(sym, &layout) .to_symbol_string(sym, &env.interns); + let fn_name = if env.exposed_to_host.contains(&sym) { + format!("roc_{}_exposed", base_name) + } else { + base_name + }; + let section_id = output.add_section( output.segment_name(StandardSegment::Text).to_vec(), format!(".text.{:x}", sym.as_u64()).as_bytes().to_vec(), @@ -298,6 +304,7 @@ fn build_object<'a, B: Backend<'a>>( return Err(format!("failed to find fn symbol for {:?}", name)); } } + Relocation::JmpToReturn { .. } => unreachable!(), }; relocations.push((section_id, elfreloc)); } diff --git a/compiler/gen_dev/tests/dev_num.rs b/compiler/gen_dev/tests/dev_num.rs index d1c0ae0d75..c82d2ca23f 100644 --- a/compiler/gen_dev/tests/dev_num.rs +++ b/compiler/gen_dev/tests/dev_num.rs @@ -1,12 +1,6 @@ -#[macro_use] -extern crate pretty_assertions; - #[macro_use] extern crate indoc; -extern crate bumpalo; -extern crate libc; - #[macro_use] mod helpers; @@ -281,6 +275,24 @@ mod dev_num { ); } + #[test] + fn gen_fast_fib_fn() { + assert_evals_to!( + indoc!( + r#" + fib = \n, a, b -> + if n == 0 then + a + else + fib (n - 1) b (a + b) + fib 10 0 1 + "# + ), + 55, + i64 + ); + } + #[test] fn f64_abs() { assert_evals_to!("Num.abs -4.7", 4.7, f64); @@ -580,18 +592,18 @@ mod dev_num { // assert_evals_to!("0.0 >= 0.0", true, bool); // } - // #[test] - // fn gen_order_of_arithmetic_ops() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 + 3 * 7 - 2 - // "# - // ), - // 20, - // i64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops() { + assert_evals_to!( + indoc!( + r#" + 1 + 3 * 7 - 2 + "# + ), + 20, + i64 + ); + } // #[test] // fn gen_order_of_arithmetic_ops_complex_float() { @@ -606,59 +618,59 @@ mod dev_num { // ); // } - // #[test] - // fn if_guard_bind_variable_false() { - // assert_evals_to!( - // indoc!( - // r#" - // wrapper = \{} -> - // when 10 is - // x if x == 5 -> 0 - // _ -> 42 + #[test] + fn if_guard_bind_variable_false() { + assert_evals_to!( + indoc!( + r#" + wrapper = \{} -> + when 10 is + x if x == 5 -> 0 + _ -> 42 - // wrapper {} - // "# - // ), - // 42, - // i64 - // ); - // } + wrapper {} + "# + ), + 42, + i64 + ); + } - // #[test] - // fn if_guard_bind_variable_true() { - // assert_evals_to!( - // indoc!( - // r#" - // wrapper = \{} -> - // when 10 is - // x if x == 10 -> 42 - // _ -> 0 + #[test] + fn if_guard_bind_variable_true() { + assert_evals_to!( + indoc!( + r#" + wrapper = \{} -> + when 10 is + x if x == 10 -> 42 + _ -> 0 - // wrapper {} - // "# - // ), - // 42, - // i64 - // ); - // } + wrapper {} + "# + ), + 42, + i64 + ); + } - // #[test] - // fn tail_call_elimination() { - // assert_evals_to!( - // indoc!( - // r#" - // sum = \n, accum -> - // when n is - // 0 -> accum - // _ -> sum (n - 1) (n + accum) + #[test] + fn tail_call_elimination() { + assert_evals_to!( + indoc!( + r#" + sum = \n, accum -> + when n is + 0 -> accum + _ -> sum (n - 1) (n + accum) - // sum 1_000_000 0 - // "# - // ), - // 500000500000, - // i64 - // ); - // } + sum 1_000_000 0 + "# + ), + 500000500000, + i64 + ); + } // #[test] // fn int_negate() { diff --git a/compiler/gen_dev/tests/dev_str.rs b/compiler/gen_dev/tests/dev_str.rs new file mode 100644 index 0000000000..19da650a89 --- /dev/null +++ b/compiler/gen_dev/tests/dev_str.rs @@ -0,0 +1,954 @@ +// #[macro_use] +// extern crate indoc; + +#[macro_use] +mod helpers; + +#[cfg(all(test, any(target_os = "linux", target_os = "macos"), any(target_arch = "x86_64"/*, target_arch = "aarch64"*/)))] +mod dev_str { + // use roc_std::{RocList, RocStr}; + // #[test] + // fn str_split_bigger_delimiter_small_str() { + // assert_evals_to!( + // indoc!( + // r#" + // List.len (Str.split "hello" "JJJJ there") + // "# + // ), + // 1, + // i64 + // ); + + // assert_evals_to!( + // indoc!( + // r#" + // when List.first (Str.split "JJJ" "JJJJ there") is + // Ok str -> + // Str.countGraphemes str + + // _ -> + // -1 + + // "# + // ), + // 3, + // i64 + // ); + // } + + // #[test] + // fn str_split_str_concat_repeated() { + // assert_evals_to!( + // indoc!( + // r#" + // when List.first (Str.split "JJJJJ" "JJJJ there") is + // Ok str -> + // str + // |> Str.concat str + // |> Str.concat str + // |> Str.concat str + // |> Str.concat str + + // _ -> + // "Not Str!" + + // "# + // ), + // RocStr::from_slice(b"JJJJJJJJJJJJJJJJJJJJJJJJJ"), + // RocStr + // ); + // } + + // #[test] + // fn str_split_small_str_bigger_delimiter() { + // assert_evals_to!( + // indoc!( + // r#" + // when + // List.first + // (Str.split "JJJ" "0123456789abcdefghi") + // is + // Ok str -> str + // _ -> "" + // "# + // ), + // RocStr::from_slice(b"JJJ"), + // RocStr + // ); + // } + + // #[test] + // fn str_split_big_str_small_delimiter() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split "01234567789abcdefghi?01234567789abcdefghi" "?" + // "# + // ), + // RocList::from_slice(&[ + // RocStr::from_slice(b"01234567789abcdefghi"), + // RocStr::from_slice(b"01234567789abcdefghi") + // ]), + // RocList + // ); + + // assert_evals_to!( + // indoc!( + // r#" + // Str.split "01234567789abcdefghi 3ch 01234567789abcdefghi" "3ch" + // "# + // ), + // RocList::from_slice(&[ + // RocStr::from_slice(b"01234567789abcdefghi "), + // RocStr::from_slice(b" 01234567789abcdefghi") + // ]), + // RocList + // ); + // } + + // #[test] + // fn str_split_small_str_small_delimiter() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split "J!J!J" "!" + // "# + // ), + // RocList::from_slice(&[ + // RocStr::from_slice(b"J"), + // RocStr::from_slice(b"J"), + // RocStr::from_slice(b"J") + // ]), + // RocList + // ); + // } + + // #[test] + // fn str_split_bigger_delimiter_big_strs() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split + // "string to split is shorter" + // "than the delimiter which happens to be very very long" + // "# + // ), + // RocList::from_slice(&[RocStr::from_slice(b"string to split is shorter")]), + // RocList + // ); + // } + + // #[test] + // fn str_split_empty_strs() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split "" "" + // "# + // ), + // RocList::from_slice(&[RocStr::from_slice(b"")]), + // RocList + // ); + // } + + // #[test] + // fn str_split_minimal_example() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split "a," "," + // "# + // ), + // RocList::from_slice(&[RocStr::from_slice(b"a"), RocStr::from_slice(b"")]), + // RocList + // ) + // } + + // #[test] + // fn str_split_small_str_big_delimiter() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split + // "1---- ---- ---- ---- ----2---- ---- ---- ---- ----" + // "---- ---- ---- ---- ----" + // |> List.len + // "# + // ), + // 3, + // i64 + // ); + + // assert_evals_to!( + // indoc!( + // r#" + // Str.split + // "1---- ---- ---- ---- ----2---- ---- ---- ---- ----" + // "---- ---- ---- ---- ----" + // "# + // ), + // RocList::from_slice(&[ + // RocStr::from_slice(b"1"), + // RocStr::from_slice(b"2"), + // RocStr::from_slice(b"") + // ]), + // RocList + // ); + // } + + // #[test] + // fn str_split_small_str_20_char_delimiter() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.split + // "3|-- -- -- -- -- -- |4|-- -- -- -- -- -- |" + // "|-- -- -- -- -- -- |" + // "# + // ), + // RocList::from_slice(&[ + // RocStr::from_slice(b"3"), + // RocStr::from_slice(b"4"), + // RocStr::from_slice(b"") + // ]), + // RocList + // ); + // } + + // #[test] + // fn str_concat_big_to_big() { + // assert_evals_to!( + // indoc!( + // r#" + // Str.concat + // "First string that is fairly long. Longer strings make for different errors. " + // "Second string that is also fairly long. Two long strings test things that might not appear with short strings." + // "# + // ), + // RocStr::from_slice(b"First string that is fairly long. Longer strings make for different errors. Second string that is also fairly long. Two long strings test things that might not appear with short strings."), + // RocStr + // ); + // } + + #[test] + fn small_str_literal() { + assert_evals_to!( + "\"JJJJJJJJJJJJJJJ\"", + [ + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0b1000_1111 + ], + [u8; 16] + ); + } + + // #[test] + // fn small_str_zeroed_literal() { + // // Verifies that we zero out unused bytes in the string. + // // This is important so that string equality tests don't randomly + // // fail due to unused memory being there! + // assert_evals_to!( + // "\"J\"", + // [ + // 0x4a, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0x00, + // 0b1000_0001 + // ], + // [u8; 16] + // ); + // } + + #[test] + fn small_str_concat_empty_first_arg() { + assert_evals_to!( + r#"Str.concat "" "JJJJJJJJJJJJJJJ""#, + [ + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0b1000_1111 + ], + [u8; 16] + ); + } + + #[test] + fn small_str_concat_empty_second_arg() { + assert_evals_to!( + r#"Str.concat "JJJJJJJJJJJJJJJ" """#, + [ + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0b1000_1111 + ], + [u8; 16] + ); + } + + // #[test] + // fn small_str_concat_small_to_big() { + // assert_evals_to!( + // r#"Str.concat "abc" " this is longer than 15 chars""#, + // RocStr::from_slice(b"abc this is longer than 15 chars"), + // RocStr + // ); + // } + + #[test] + fn small_str_concat_small_to_small_staying_small() { + assert_evals_to!( + r#"Str.concat "J" "JJJJJJJJJJJJJJ""#, + [ + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0x4a, + 0b1000_1111 + ], + [u8; 16] + ); + } + + // #[test] + // fn small_str_concat_small_to_small_overflow_to_big() { + // assert_evals_to!( + // r#"Str.concat "abcdefghijklm" "nopqrstuvwxyz""#, + // RocStr::from_slice(b"abcdefghijklmnopqrstuvwxyz"), + // RocStr + // ); + // } + + // #[test] + // fn str_concat_empty() { + // assert_evals_to!(r#"Str.concat "" """#, RocStr::default(), RocStr); + // } + + // #[test] + // fn small_str_is_empty() { + // assert_evals_to!(r#"Str.isEmpty "abc""#, false, bool); + // } + + // #[test] + // fn big_str_is_empty() { + // assert_evals_to!( + // r#"Str.isEmpty "this is more than 15 chars long""#, + // false, + // bool + // ); + // } + + // #[test] + // fn empty_str_is_empty() { + // assert_evals_to!(r#"Str.isEmpty """#, true, bool); + // } + + // #[test] + // fn str_starts_with() { + // assert_evals_to!(r#"Str.startsWith "hello world" "hell""#, true, bool); + // assert_evals_to!(r#"Str.startsWith "hello world" """#, true, bool); + // assert_evals_to!(r#"Str.startsWith "nope" "hello world""#, false, bool); + // assert_evals_to!(r#"Str.startsWith "hell" "hello world""#, false, bool); + // assert_evals_to!(r#"Str.startsWith "" "hello world""#, false, bool); + // } + + // #[test] + // fn str_starts_with_code_point() { + // assert_evals_to!( + // &format!(r#"Str.startsWithCodePt "foobar" {}"#, 'f' as u32), + // true, + // bool + // ); + // assert_evals_to!( + // &format!(r#"Str.startsWithCodePt "zoobar" {}"#, 'f' as u32), + // false, + // bool + // ); + // } + + // #[test] + // fn str_ends_with() { + // assert_evals_to!(r#"Str.endsWith "hello world" "world""#, true, bool); + // assert_evals_to!(r#"Str.endsWith "nope" "hello world""#, false, bool); + // assert_evals_to!(r#"Str.endsWith "" "hello world""#, false, bool); + // } + + // #[test] + // fn str_count_graphemes_small_str() { + // assert_evals_to!(r#"Str.countGraphemes "å🤔""#, 2, usize); + // } + + // #[test] + // fn str_count_graphemes_three_js() { + // assert_evals_to!(r#"Str.countGraphemes "JJJ""#, 3, usize); + // } + + // #[test] + // fn str_count_graphemes_big_str() { + // assert_evals_to!( + // r#"Str.countGraphemes "6🤔å🤔e¥🤔çppkd🙃1jdal🦯asdfa∆ltråø˚waia8918.,🏅jjc""#, + // 45, + // usize + // ); + // } + + // #[test] + // fn str_starts_with_same_big_str() { + // assert_evals_to!( + // r#"Str.startsWith "123456789123456789" "123456789123456789""#, + // true, + // bool + // ); + // } + + // #[test] + // fn str_starts_with_different_big_str() { + // assert_evals_to!( + // r#"Str.startsWith "12345678912345678910" "123456789123456789""#, + // true, + // bool + // ); + // } + + // #[test] + // fn str_starts_with_same_small_str() { + // assert_evals_to!(r#"Str.startsWith "1234" "1234""#, true, bool); + // } + + // #[test] + // fn str_starts_with_different_small_str() { + // assert_evals_to!(r#"Str.startsWith "1234" "12""#, true, bool); + // } + // #[test] + // fn str_starts_with_false_small_str() { + // assert_evals_to!(r#"Str.startsWith "1234" "23""#, false, bool); + // } + + // #[test] + // fn str_from_int() { + // assert_evals_to!( + // r#"Str.fromInt 1234"#, + // roc_std::RocStr::from_slice("1234".as_bytes()), + // roc_std::RocStr + // ); + // assert_evals_to!( + // r#"Str.fromInt 0"#, + // roc_std::RocStr::from_slice("0".as_bytes()), + // roc_std::RocStr + // ); + // assert_evals_to!( + // r#"Str.fromInt -1"#, + // roc_std::RocStr::from_slice("-1".as_bytes()), + // roc_std::RocStr + // ); + + // let max = format!("{}", i64::MAX); + // assert_evals_to!( + // r#"Str.fromInt Num.maxInt"#, + // RocStr::from_slice(max.as_bytes()), + // RocStr + // ); + + // let min = format!("{}", i64::MIN); + // assert_evals_to!( + // r#"Str.fromInt Num.minInt"#, + // RocStr::from_slice(min.as_bytes()), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_single_ascii() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97 ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_many_ascii() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 98, 99, 0x7E ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("abc~".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_single_unicode() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 0xE2, 0x88, 0x86 ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("∆".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_many_unicode() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 0xE2, 0x88, 0x86, 0xC5, 0x93, 0xC2, 0xAC ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("∆œ¬".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_single_grapheme() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 0xF0, 0x9F, 0x92, 0x96 ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("💖".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_many_grapheme() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 0xF0, 0x9F, 0x92, 0x96, 0xF0, 0x9F, 0xA4, 0xA0, 0xF0, 0x9F, 0x9A, 0x80 ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("💖🤠🚀".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_pass_all() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 0xF0, 0x9F, 0x92, 0x96, 98, 0xE2, 0x88, 0x86 ] is + // Ok val -> val + // Err _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("💖b∆".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_invalid_start_byte() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 98, 0x80, 99 ] is + // Err (BadUtf8 InvalidStartByte byteIndex) -> + // if byteIndex == 2 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_unexpected_end_of_sequence() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 98, 99, 0xC2 ] is + // Err (BadUtf8 UnexpectedEndOfSequence byteIndex) -> + // if byteIndex == 3 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_expected_continuation() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 98, 99, 0xC2, 0x00 ] is + // Err (BadUtf8 ExpectedContinuation byteIndex) -> + // if byteIndex == 3 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_overlong_encoding() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 0xF0, 0x80, 0x80, 0x80 ] is + // Err (BadUtf8 OverlongEncoding byteIndex) -> + // if byteIndex == 1 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_codepoint_too_large() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 0xF4, 0x90, 0x80, 0x80 ] is + // Err (BadUtf8 CodepointTooLarge byteIndex) -> + // if byteIndex == 1 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_fail_surrogate_half() { + // assert_evals_to!( + // indoc!( + // r#" + // when Str.fromUtf8 [ 97, 98, 0xED, 0xA0, 0x80 ] is + // Err (BadUtf8 EncodesSurrogateHalf byteIndex) -> + // if byteIndex == 2 then + // "a" + // else + // "b" + // _ -> "" + // "# + // ), + // roc_std::RocStr::from_slice("a".as_bytes()), + // roc_std::RocStr + // ); + // } + + // #[test] + // fn str_equality() { + // assert_evals_to!(r#""a" == "a""#, true, bool); + // assert_evals_to!( + // r#""loremipsumdolarsitamet" == "loremipsumdolarsitamet""#, + // true, + // bool + // ); + // assert_evals_to!(r#""a" != "b""#, true, bool); + // assert_evals_to!(r#""a" == "b""#, false, bool); + // } + + // #[test] + // fn str_clone() { + // use roc_std::RocStr; + // let long = RocStr::from_slice("loremipsumdolarsitamet".as_bytes()); + // let short = RocStr::from_slice("x".as_bytes()); + // let empty = RocStr::from_slice("".as_bytes()); + + // debug_assert_eq!(long.clone(), long); + // debug_assert_eq!(short.clone(), short); + // debug_assert_eq!(empty.clone(), empty); + // } + + // #[test] + // fn nested_recursive_literal() { + // assert_evals_to!( + // indoc!( + // r#" + // Expr : [ Add Expr Expr, Val I64, Var I64 ] + + // expr : Expr + // expr = Add (Add (Val 3) (Val 1)) (Add (Val 1) (Var 1)) + + // printExpr : Expr -> Str + // printExpr = \e -> + // when e is + // Add a b -> + // "Add (" + // |> Str.concat (printExpr a) + // |> Str.concat ") (" + // |> Str.concat (printExpr b) + // |> Str.concat ")" + // Val v -> "Val " |> Str.concat (Str.fromInt v) + // Var v -> "Var " |> Str.concat (Str.fromInt v) + + // printExpr expr + // "# + // ), + // RocStr::from_slice(b"Add (Add (Val 3) (Val 1)) (Add (Val 1) (Var 1))"), + // RocStr + // ); + // } + + // #[test] + // fn str_join_comma_small() { + // assert_evals_to!( + // r#"Str.joinWith ["1", "2"] ", " "#, + // RocStr::from("1, 2"), + // RocStr + // ); + // } + + // #[test] + // fn str_join_comma_big() { + // assert_evals_to!( + // r#"Str.joinWith ["10000000", "2000000", "30000000"] ", " "#, + // RocStr::from("10000000, 2000000, 30000000"), + // RocStr + // ); + // } + + // #[test] + // fn str_join_comma_single() { + // assert_evals_to!(r#"Str.joinWith ["1"] ", " "#, RocStr::from("1"), RocStr); + // } + + // #[test] + // fn str_from_float() { + // assert_evals_to!(r#"Str.fromFloat 3.14"#, RocStr::from("3.14"), RocStr); + // } + + // #[test] + // fn str_to_utf8() { + // assert_evals_to!( + // r#"Str.toUtf8 "hello""#, + // RocList::from_slice(&[104, 101, 108, 108, 111]), + // RocList + // ); + // assert_evals_to!( + // r#"Str.toUtf8 "this is a long string""#, + // RocList::from_slice(&[ + // 116, 104, 105, 115, 32, 105, 115, 32, 97, 32, 108, 111, 110, 103, 32, 115, 116, + // 114, 105, 110, 103 + // ]), + // RocList + // ); + // } + + // #[test] + // fn str_from_utf8_range() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { count: 5, start: 0 } is + // Ok utf8String -> utf8String + // _ -> "" + // "# + // ), + // RocStr::from("hello"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_slice() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { count: 4, start: 1 } is + // Ok utf8String -> utf8String + // _ -> "" + // "# + // ), + // RocStr::from("ello"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_slice_not_end() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { count: 3, start: 1 } is + // Ok utf8String -> utf8String + // _ -> "" + // "# + // ), + // RocStr::from("ell"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_order_does_not_matter() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { start: 1, count: 3 } is + // Ok utf8String -> utf8String + // _ -> "" + // "# + // ), + // RocStr::from("ell"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_out_of_bounds_start_value() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { start: 7, count: 3 } is + // Ok _ -> "" + // Err (BadUtf8 _ _) -> "" + // Err OutOfBounds -> "out of bounds" + // "# + // ), + // RocStr::from("out of bounds"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_count_too_high() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { start: 0, count: 6 } is + // Ok _ -> "" + // Err (BadUtf8 _ _) -> "" + // Err OutOfBounds -> "out of bounds" + // "# + // ), + // RocStr::from("out of bounds"), + // RocStr + // ); + // } + + // #[test] + // fn str_from_utf8_range_count_too_high_for_start() { + // assert_evals_to!( + // indoc!( + // r#" + // bytes = Str.toUtf8 "hello" + // when Str.fromUtf8Range bytes { start: 4, count: 3 } is + // Ok _ -> "" + // Err (BadUtf8 _ _) -> "" + // Err OutOfBounds -> "out of bounds" + // "# + // ), + // RocStr::from("out of bounds"), + // RocStr + // ); + // } +} diff --git a/compiler/gen_dev/tests/helpers/eval.rs b/compiler/gen_dev/tests/helpers/eval.rs index 675f981e72..84e5dfdf22 100644 --- a/compiler/gen_dev/tests/helpers/eval.rs +++ b/compiler/gen_dev/tests/helpers/eval.rs @@ -94,10 +94,12 @@ pub fn helper<'a>( let main_fn_layout = loaded.entry_point.layout; let mut layout_ids = roc_mono::layout::LayoutIds::default(); - let main_fn_name = layout_ids + let main_fn_name_base = layout_ids .get_toplevel(main_fn_symbol, &main_fn_layout) .to_symbol_string(main_fn_symbol, &interns); + let main_fn_name = format!("roc_{}_exposed", main_fn_name_base); + let mut lines = Vec::new(); // errors whose reporting we delay (so we can see that code gen generates runtime errors) let mut delayed_errors = Vec::new(); @@ -143,12 +145,13 @@ pub fn helper<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/compiler/gen_llvm/src/llvm/bitcode.rs b/compiler/gen_llvm/src/llvm/bitcode.rs index f873af0518..52b604dff3 100644 --- a/compiler/gen_llvm/src/llvm/bitcode.rs +++ b/compiler/gen_llvm/src/llvm/bitcode.rs @@ -10,7 +10,7 @@ use inkwell::types::{BasicType, BasicTypeEnum}; use inkwell::values::{BasicValue, BasicValueEnum, CallSiteValue, FunctionValue, InstructionValue}; use inkwell::AddressSpace; use roc_module::symbol::Symbol; -use roc_mono::layout::{Layout, LayoutIds, UnionLayout}; +use roc_mono::layout::{LambdaSet, Layout, LayoutIds, UnionLayout}; pub fn call_bitcode_fn<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, @@ -189,7 +189,7 @@ fn build_has_tag_id_help<'a, 'ctx, 'env>( pub fn build_transform_caller<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, argument_layouts: &[Layout<'a>], ) -> FunctionValue<'ctx> { let fn_name: &str = &format!( @@ -212,7 +212,7 @@ pub fn build_transform_caller<'a, 'ctx, 'env>( fn build_transform_caller_help<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, roc_function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, argument_layouts: &[Layout<'a>], fn_name: &str, ) -> FunctionValue<'ctx> { @@ -270,7 +270,7 @@ fn build_transform_caller_help<'a, 'ctx, 'env>( arguments_cast.push(argument); } - match closure_data_layout { + match closure_data_layout.runtime_representation() { Layout::Struct(&[]) => { // nothing to add } @@ -529,7 +529,7 @@ pub fn build_eq_wrapper<'a, 'ctx, 'env>( pub fn build_compare_wrapper<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, roc_function: FunctionValue<'ctx>, - closure_data_layout: Layout<'a>, + closure_data_layout: LambdaSet<'a>, layout: &Layout<'a>, ) -> FunctionValue<'ctx> { let block = env.builder.get_insert_block().expect("to be in a function"); @@ -595,7 +595,7 @@ pub fn build_compare_wrapper<'a, 'ctx, 'env>( let default = [value1, value2]; - let arguments_cast = match closure_data_layout { + let arguments_cast = match closure_data_layout.runtime_representation() { Layout::Struct(&[]) => { // nothing to add &default diff --git a/compiler/gen_llvm/src/llvm/build.rs b/compiler/gen_llvm/src/llvm/build.rs index 2d7796e3a0..9864f277a8 100644 --- a/compiler/gen_llvm/src/llvm/build.rs +++ b/compiler/gen_llvm/src/llvm/build.rs @@ -15,7 +15,7 @@ use crate::llvm::build_list::{ }; use crate::llvm::build_str::{ empty_str, str_concat, str_count_graphemes, str_ends_with, str_from_float, str_from_int, - str_from_utf8, str_from_utf8_range, str_join_with, str_number_of_bytes, str_split, + str_from_utf8, str_from_utf8_range, str_join_with, str_number_of_bytes, str_repeat, str_split, str_starts_with, str_starts_with_code_point, str_to_utf8, }; use crate::llvm::compare::{generic_eq, generic_neq}; @@ -638,7 +638,7 @@ pub fn construct_optimization_passes<'a>( let pmb = PassManagerBuilder::create(); match opt_level { - OptLevel::Normal => { + OptLevel::Development | OptLevel::Normal => { pmb.set_optimization_level(OptimizationLevel::None); } OptLevel::Optimize => { @@ -704,8 +704,14 @@ fn promote_to_main_function<'a, 'ctx, 'env>( let main_fn_name = "$Test.main"; // Add main to the module. - let main_fn = - expose_function_to_host_help_c_abi(env, main_fn_name, roc_main_fn, &[], main_fn_name); + let main_fn = expose_function_to_host_help_c_abi( + env, + main_fn_name, + roc_main_fn, + &[], + top_level.result, + main_fn_name, + ); (main_fn_name, main_fn) } @@ -1164,8 +1170,16 @@ pub fn build_exp_expr<'a, 'ctx, 'env>( StructAtIndex { index, structure, .. } => { + let (value, layout) = load_symbol_and_layout(scope, structure); + + let layout = if let Layout::LambdaSet(lambda_set) = layout { + lambda_set.runtime_representation() + } else { + *layout + }; + // extract field from a record - match load_symbol_and_layout(scope, structure) { + match (value, layout) { (StructValue(argument), Layout::Struct(fields)) => { debug_assert!(!fields.is_empty()); env.builder @@ -2601,6 +2615,18 @@ pub fn load_symbol_and_layout<'a, 'ctx, 'b>( None => panic!("There was no entry for {:?} in scope {:?}", symbol, scope), } } + +pub fn load_symbol_and_lambda_set<'a, 'ctx, 'b>( + scope: &'b Scope<'a, 'ctx>, + symbol: &Symbol, +) -> (BasicValueEnum<'ctx>, LambdaSet<'a>) { + match scope.get(symbol) { + Some((Layout::LambdaSet(lambda_set), ptr)) => (*ptr, *lambda_set), + Some((other, ptr)) => panic!("Not a lambda set: {:?}, {:?}", other, ptr), + None => panic!("There was no entry for {:?} in scope {:?}", symbol, scope), + } +} + fn access_index_struct_value<'ctx>( builder: &Builder<'ctx>, from_value: StructValue<'ctx>, @@ -3055,6 +3081,7 @@ fn expose_function_to_host<'a, 'ctx, 'env>( symbol: Symbol, roc_function: FunctionValue<'ctx>, arguments: &[Layout<'a>], + return_layout: Layout<'a>, ) { // Assumption: there is only one specialization of a host-exposed function let ident_string = symbol.as_str(&env.interns); @@ -3065,26 +3092,19 @@ fn expose_function_to_host<'a, 'ctx, 'env>( ident_string, roc_function, arguments, + return_layout, &c_function_name, ); } -fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( +fn expose_function_to_host_help_c_abi_generic<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, - ident_string: &str, roc_function: FunctionValue<'ctx>, arguments: &[Layout<'a>], c_function_name: &str, ) -> FunctionValue<'ctx> { - let context = env.context; - - let wrapper_return_type = context.struct_type( - &[ - context.i64_type().into(), - roc_function.get_type().get_return_type().unwrap(), - ], - false, - ); + // NOTE we ingore env.is_gen_test here + let wrapper_return_type = roc_function.get_type().get_return_type().unwrap(); let mut cc_argument_types = Vec::with_capacity_in(arguments.len(), env.arena); for layout in arguments { @@ -3095,6 +3115,7 @@ fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( // let mut argument_types = roc_function.get_type().get_param_types(); let mut argument_types = cc_argument_types; let return_type = wrapper_return_type; + let output_type = return_type.ptr_type(AddressSpace::Generic); argument_types.push(output_type.into()); @@ -3122,9 +3143,11 @@ fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( debug_info_init!(env, c_function); // drop the final argument, which is the pointer we write the result into - let args = c_function.get_params(); - let output_arg_index = args.len() - 1; - let args = &args[..args.len() - 1]; + let args_vector = c_function.get_params(); + let mut args = args_vector.as_slice(); + let args_length = args.len(); + + args = &args[..args.len() - 1]; let mut arguments_for_call = Vec::with_capacity_in(args.len(), env.arena); @@ -3169,19 +3192,187 @@ fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( let call_unwrapped_result = call_unwrapped.try_as_basic_value().left().unwrap(); - make_good_roc_result(env, call_unwrapped_result) + // make_good_roc_result(env, call_unwrapped_result) + call_unwrapped_result } }; + let output_arg_index = args_length - 1; + let output_arg = c_function .get_nth_param(output_arg_index as u32) .unwrap() .into_pointer_value(); builder.build_store(output_arg, call_result); - builder.build_return(None); + c_function +} + +fn expose_function_to_host_help_c_abi<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + ident_string: &str, + roc_function: FunctionValue<'ctx>, + arguments: &[Layout<'a>], + return_layout: Layout<'a>, + c_function_name: &str, +) -> FunctionValue<'ctx> { + let context = env.context; + + // a generic version that writes the result into a passed *u8 pointer + if !env.is_gen_test { + expose_function_to_host_help_c_abi_generic( + env, + roc_function, + arguments, + &format!("{}_generic", c_function_name), + ); + } + + let wrapper_return_type = if env.is_gen_test { + context + .struct_type( + &[ + context.i64_type().into(), + roc_function.get_type().get_return_type().unwrap(), + ], + false, + ) + .into() + } else { + roc_function.get_type().get_return_type().unwrap() + }; + + let mut cc_argument_types = Vec::with_capacity_in(arguments.len(), env.arena); + for layout in arguments { + cc_argument_types.push(to_cc_type(env, layout)); + } + + // STEP 1: turn `f : a,b,c -> d` into `f : a,b,c, &d -> {}` if the C abi demands it + let mut argument_types = cc_argument_types; + let return_type = wrapper_return_type; + + let cc_return = to_cc_return(env, &return_layout); + + let c_function_type = match cc_return { + CCReturn::Void if !env.is_gen_test => { + env.context.void_type().fn_type(&argument_types, false) + } + CCReturn::Return if !env.is_gen_test => return_type.fn_type(&argument_types, false), + _ => { + let output_type = return_type.ptr_type(AddressSpace::Generic); + argument_types.push(output_type.into()); + env.context.void_type().fn_type(&argument_types, false) + } + }; + + let c_function = add_func( + env.module, + c_function_name, + c_function_type, + Linkage::External, + C_CALL_CONV, + ); + + let subprogram = env.new_subprogram(c_function_name); + c_function.set_subprogram(subprogram); + + // STEP 2: build the exposed function's body + let builder = env.builder; + let context = env.context; + + let entry = context.append_basic_block(c_function, "entry"); + + builder.position_at_end(entry); + + debug_info_init!(env, c_function); + + // drop the final argument, which is the pointer we write the result into + let args_vector = c_function.get_params(); + let mut args = args_vector.as_slice(); + let args_length = args.len(); + + match cc_return { + CCReturn::Return if !env.is_gen_test => { + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + CCReturn::Void if !env.is_gen_test => { + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + _ => { + args = &args[..args.len() - 1]; + debug_assert_eq!(args.len(), roc_function.get_params().len()); + } + } + + let mut arguments_for_call = Vec::with_capacity_in(args.len(), env.arena); + + let it = args.iter().zip(roc_function.get_type().get_param_types()); + for (arg, fastcc_type) in it { + let arg_type = arg.get_type(); + if arg_type == fastcc_type { + // the C and Fast calling conventions agree + arguments_for_call.push(*arg); + } else { + let cast = complex_bitcast_check_size(env, *arg, fastcc_type, "to_fastcc_type"); + arguments_for_call.push(cast); + } + } + + let arguments_for_call = &arguments_for_call.into_bump_slice(); + + let call_result = { + if env.is_gen_test { + let roc_wrapper_function = make_exception_catcher(env, roc_function); + debug_assert_eq!( + arguments_for_call.len(), + roc_wrapper_function.get_params().len() + ); + + builder.position_at_end(entry); + + let call_wrapped = builder.build_call( + roc_wrapper_function, + arguments_for_call, + "call_wrapped_function", + ); + call_wrapped.set_call_convention(FAST_CALL_CONV); + + call_wrapped.try_as_basic_value().left().unwrap() + } else { + let call_unwrapped = + builder.build_call(roc_function, arguments_for_call, "call_unwrapped_function"); + call_unwrapped.set_call_convention(FAST_CALL_CONV); + + let call_unwrapped_result = call_unwrapped.try_as_basic_value().left().unwrap(); + + // make_good_roc_result(env, call_unwrapped_result) + call_unwrapped_result + } + }; + + match cc_return { + CCReturn::Void if !env.is_gen_test => { + // TODO return empty struct here? + builder.build_return(None); + } + CCReturn::Return if !env.is_gen_test => { + builder.build_return(Some(&call_result)); + } + _ => { + let output_arg_index = args_length - 1; + + let output_arg = c_function + .get_nth_param(output_arg_index as u32) + .unwrap() + .into_pointer_value(); + + builder.build_store(output_arg, call_result); + builder.build_return(None); + } + } + // STEP 3: build a {} -> u64 function that gives the size of the return type let size_function_type = env.context.i64_type().fn_type(&[], false); let size_function_name: String = format!("roc__{}_size", ident_string); @@ -3695,7 +3886,13 @@ fn build_proc_header<'a, 'ctx, 'env>( if env.exposed_to_host.contains(&symbol) { let arguments = Vec::from_iter_in(proc.args.iter().map(|(layout, _)| *layout), env.arena); - expose_function_to_host(env, symbol, fn_val, arguments.into_bump_slice()); + expose_function_to_host( + env, + symbol, + fn_val, + arguments.into_bump_slice(), + proc.ret_layout, + ); } fn_val @@ -3762,23 +3959,17 @@ pub fn build_closure_caller<'a, 'ctx, 'env>( builder.position_at_end(entry); - let mut parameters = function_value.get_params(); - let output = parameters.pop().unwrap().into_pointer_value(); + let mut evaluator_arguments = function_value.get_params(); - let closure_data = if let Some(closure_data_ptr) = parameters.pop() { - let closure_data = - builder.build_load(closure_data_ptr.into_pointer_value(), "load_closure_data"); + // the final parameter is the output pointer, pop it + let output = evaluator_arguments.pop().unwrap().into_pointer_value(); - env.arena.alloc([closure_data]) as &[_] - } else { - &[] - }; - - let mut parameters = parameters; - - for param in parameters.iter_mut() { - debug_assert!(param.is_pointer_value()); - *param = builder.build_load(param.into_pointer_value(), "load_param"); + // NOTE this may be incorrect in the long run + // here we load any argument that is a pointer + for param in evaluator_arguments.iter_mut() { + if param.is_pointer_value() { + *param = builder.build_load(param.into_pointer_value(), "load_param"); + } } let call_result = if env.is_gen_test { @@ -3787,13 +3978,13 @@ pub fn build_closure_caller<'a, 'ctx, 'env>( function_value, evaluator, evaluator.get_call_conventions(), - closure_data, + &evaluator_arguments, result_type, ) } else { let call = env .builder - .build_call(evaluator, closure_data, "call_function"); + .build_call(evaluator, &evaluator_arguments, "call_function"); call.set_call_convention(evaluator.get_call_conventions()); @@ -4094,7 +4285,7 @@ fn roc_function_call<'a, 'ctx, 'env>( layout_ids: &mut LayoutIds<'a>, transform: FunctionValue<'ctx>, closure_data: BasicValueEnum<'ctx>, - closure_data_layout: Layout<'a>, + lambda_set: LambdaSet<'a>, closure_data_is_owned: bool, argument_layouts: &[Layout<'a>], ) -> RocFunctionCall<'ctx> { @@ -4105,15 +4296,15 @@ fn roc_function_call<'a, 'ctx, 'env>( .build_alloca(closure_data.get_type(), "closure_data_ptr"); env.builder.build_store(closure_data_ptr, closure_data); - let stepper_caller = - build_transform_caller(env, transform, closure_data_layout, argument_layouts) - .as_global_value() - .as_pointer_value(); - - let inc_closure_data = build_inc_n_wrapper(env, layout_ids, &closure_data_layout) + let stepper_caller = build_transform_caller(env, transform, lambda_set, argument_layouts) .as_global_value() .as_pointer_value(); + let inc_closure_data = + build_inc_n_wrapper(env, layout_ids, &lambda_set.runtime_representation()) + .as_global_value() + .as_pointer_value(); + let closure_data_is_owned = env .context .bool_type() @@ -4167,7 +4358,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match list_layout { Layout::Builtin(Builtin::EmptyList) => default, @@ -4179,7 +4370,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4209,7 +4400,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (Layout::Builtin(Builtin::EmptyList), _) => empty_list(env), @@ -4224,7 +4415,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4241,7 +4432,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (list2, list2_layout) = load_symbol_and_layout(scope, &args[1]); let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match (list1_layout, list2_layout, return_layout) { ( @@ -4256,7 +4447,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4285,7 +4476,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (list3, list3_layout) = load_symbol_and_layout(scope, &args[2]); let function = passed_function_at_index!(3); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[4]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[4]); match (list1_layout, list2_layout, list3_layout, return_layout) { ( @@ -4302,7 +4493,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4334,7 +4525,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (Layout::Builtin(Builtin::EmptyList), _) => empty_list(env), @@ -4349,7 +4540,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4367,7 +4558,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match list_layout { Layout::Builtin(Builtin::EmptyList) => empty_list(env), @@ -4379,7 +4570,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4397,7 +4588,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (_, Layout::Builtin(Builtin::EmptyList)) @@ -4413,7 +4604,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4441,7 +4632,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match (list_layout, return_layout) { (_, Layout::Builtin(Builtin::EmptyList)) @@ -4457,7 +4648,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4494,7 +4685,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let function = passed_function_at_index!(1); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[2]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[2]); match list_layout { Layout::Builtin(Builtin::EmptyList) => empty_list(env), @@ -4504,7 +4695,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let argument_layouts = &[**element_layout, **element_layout]; let compare_wrapper = - build_compare_wrapper(env, function, *closure_layout, element_layout) + build_compare_wrapper(env, function, closure_layout, element_layout) .as_global_value() .as_pointer_value(); @@ -4513,7 +4704,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4535,7 +4726,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( let (dict, dict_layout) = load_symbol_and_layout(scope, &args[0]); let (default, default_layout) = load_symbol_and_layout(scope, &args[1]); let function = passed_function_at_index!(2); - let (closure, closure_layout) = load_symbol_and_layout(scope, &args[3]); + let (closure, closure_layout) = load_symbol_and_lambda_set(scope, &args[3]); match dict_layout { Layout::Builtin(Builtin::EmptyDict) => { @@ -4550,7 +4741,7 @@ fn run_higher_order_low_level<'a, 'ctx, 'env>( layout_ids, function, closure, - *closure_layout, + closure_layout, function_owns_closure_data, argument_layouts, ); @@ -4662,6 +4853,12 @@ fn run_low_level<'a, 'ctx, 'env>( str_to_utf8(env, string.into_struct_value()) } + StrRepeat => { + // Str.repeat : Str, Nat -> Str + debug_assert_eq!(args.len(), 2); + + str_repeat(env, scope, args[0], args[1]) + } StrSplit => { // Str.split : Str, Str -> List Str debug_assert_eq!(args.len(), 2); @@ -4836,7 +5033,7 @@ fn run_low_level<'a, 'ctx, 'env>( Usize | Int128 | Int64 | Int32 | Int16 | Int8 => { build_int_unary_op(env, arg.into_int_value(), arg_builtin, op) } - Float128 | Float64 | Float32 | Float16 => { + Float128 | Float64 | Float32 => { build_float_unary_op(env, arg.into_float_value(), op) } _ => { @@ -4932,7 +5129,7 @@ fn run_low_level<'a, 'ctx, 'env>( "lt_or_gt", ) } - Float128 | Float64 | Float32 | Float16 => { + Float128 | Float64 | Float32 => { let are_equal = env.builder.build_float_compare( FloatPredicate::OEQ, lhs_arg.into_float_value(), @@ -5378,8 +5575,7 @@ fn to_cc_type_builtin<'a, 'ctx, 'env>( | Builtin::Decimal | Builtin::Float128 | Builtin::Float64 - | Builtin::Float32 - | Builtin::Float16 => basic_type_from_builtin(env, builtin), + | Builtin::Float32 => basic_type_from_builtin(env, builtin), Builtin::Str | Builtin::EmptyStr | Builtin::List(_) | Builtin::EmptyList => { env.str_list_c_abi().into() } @@ -5742,7 +5938,7 @@ pub fn build_num_binop<'a, 'ctx, 'env>( rhs_layout, op, ), - Float128 | Float64 | Float32 | Float16 => build_float_binop( + Float128 | Float64 | Float32 => build_float_binop( env, parent, lhs_arg.into_float_value(), diff --git a/compiler/gen_llvm/src/llvm/build_hash.rs b/compiler/gen_llvm/src/llvm/build_hash.rs index d673251296..d3dc006198 100644 --- a/compiler/gen_llvm/src/llvm/build_hash.rs +++ b/compiler/gen_llvm/src/llvm/build_hash.rs @@ -59,6 +59,15 @@ fn build_hash_layout<'a, 'ctx, 'env>( val.into_struct_value(), ), + Layout::LambdaSet(lambda_set) => build_hash_layout( + env, + layout_ids, + seed, + val, + &lambda_set.runtime_representation(), + when_recursive, + ), + Layout::Union(union_layout) => { build_hash_tag(env, layout_ids, layout, union_layout, seed, val) } @@ -123,7 +132,6 @@ fn hash_builtin<'a, 'ctx, 'env>( | Builtin::Float64 | Builtin::Float32 | Builtin::Float128 - | Builtin::Float16 | Builtin::Decimal | Builtin::Usize => { let hash_bytes = store_and_use_as_u8_ptr(env, val, layout); diff --git a/compiler/gen_llvm/src/llvm/build_str.rs b/compiler/gen_llvm/src/llvm/build_str.rs index 87a8c50c57..02098150b7 100644 --- a/compiler/gen_llvm/src/llvm/build_str.rs +++ b/compiler/gen_llvm/src/llvm/build_str.rs @@ -1,5 +1,5 @@ use crate::llvm::bitcode::{call_bitcode_fn, call_void_bitcode_fn}; -use crate::llvm::build::{complex_bitcast, struct_from_fields, Env, Scope}; +use crate::llvm::build::{complex_bitcast, Env, Scope}; use crate::llvm::build_list::{allocate_list, call_bitcode_fn_returns_list, store_list}; use inkwell::builder::Builder; use inkwell::values::{BasicValueEnum, FunctionValue, IntValue, PointerValue, StructValue}; @@ -12,6 +12,18 @@ use super::build::load_symbol; pub static CHAR_LAYOUT: Layout = Layout::Builtin(Builtin::Int8); +/// Str.repeat : Str, Nat -> Str +pub fn str_repeat<'a, 'ctx, 'env>( + env: &Env<'a, 'ctx, 'env>, + scope: &Scope<'a, 'ctx>, + str_symbol: Symbol, + count_symbol: Symbol, +) -> BasicValueEnum<'ctx> { + let str_c_abi = str_symbol_to_c_abi(env, scope, str_symbol); + let count = load_symbol(scope, &count_symbol); + call_bitcode_fn(env, &[str_c_abi.into(), count], bitcode::STR_REPEAT) +} + /// Str.split : Str, Str -> List Str pub fn str_split<'a, 'ctx, 'env>( env: &Env<'a, 'ctx, 'env>, @@ -268,48 +280,19 @@ fn decode_from_utf8_result<'a, 'ctx, 'env>( let ctx = env.context; let fields = match env.ptr_bytes { - 8 => [ + 8 | 4 => [ env.ptr_int().into(), super::convert::zig_str_type(env).into(), env.context.bool_type().into(), ctx.i8_type().into(), ], - 4 => [ - super::convert::zig_str_type(env).into(), - env.ptr_int().into(), - env.context.bool_type().into(), - ctx.i8_type().into(), - ], _ => unreachable!(), }; let record_type = env.context.struct_type(&fields, false); match env.ptr_bytes { - 8 => { - let zig_struct = builder - .build_load(pointer, "load_utf8_validate_bytes_result") - .into_struct_value(); - - let string = builder - .build_extract_value(zig_struct, 0, "string") - .unwrap(); - - let byte_index = builder - .build_extract_value(zig_struct, 1, "byte_index") - .unwrap(); - - let is_ok = builder.build_extract_value(zig_struct, 2, "is_ok").unwrap(); - - let problem_code = builder - .build_extract_value(zig_struct, 3, "problem_code") - .unwrap(); - - let values = [byte_index, string, is_ok, problem_code]; - - struct_from_fields(env, record_type, values.iter().copied().enumerate()) - } - 4 => { + 8 | 4 => { let result_ptr_cast = env .builder .build_bitcast( diff --git a/compiler/gen_llvm/src/llvm/compare.rs b/compiler/gen_llvm/src/llvm/compare.rs index e2fa27604c..1ca6178c0b 100644 --- a/compiler/gen_llvm/src/llvm/compare.rs +++ b/compiler/gen_llvm/src/llvm/compare.rs @@ -103,7 +103,6 @@ fn build_eq_builtin<'a, 'ctx, 'env>( Builtin::Float128 => float_cmp(FloatPredicate::OEQ, "eq_f128"), Builtin::Float64 => float_cmp(FloatPredicate::OEQ, "eq_f64"), Builtin::Float32 => float_cmp(FloatPredicate::OEQ, "eq_f32"), - Builtin::Float16 => float_cmp(FloatPredicate::OEQ, "eq_f16"), Builtin::Str => str_equal(env, lhs_val, rhs_val), Builtin::List(elem) => build_list_eq( @@ -156,6 +155,8 @@ fn build_eq<'a, 'ctx, 'env>( rhs_val.into_struct_value(), ), + Layout::LambdaSet(_) => unreachable!("cannot compare closures"), + Layout::Union(union_layout) => build_tag_eq( env, layout_ids, @@ -245,7 +246,6 @@ fn build_neq_builtin<'a, 'ctx, 'env>( Builtin::Float128 => float_cmp(FloatPredicate::ONE, "neq_f128"), Builtin::Float64 => float_cmp(FloatPredicate::ONE, "neq_f64"), Builtin::Float32 => float_cmp(FloatPredicate::ONE, "neq_f32"), - Builtin::Float16 => float_cmp(FloatPredicate::ONE, "neq_f16"), Builtin::Str => { let is_equal = str_equal(env, lhs_val, rhs_val).into_int_value(); @@ -336,6 +336,7 @@ fn build_neq<'a, 'ctx, 'env>( Layout::RecursivePointer => { unreachable!("recursion pointers should never be compared directly") } + Layout::LambdaSet(_) => unreachable!("cannot compare closure"), } } diff --git a/compiler/gen_llvm/src/llvm/convert.rs b/compiler/gen_llvm/src/llvm/convert.rs index 7192400542..d32216c10a 100644 --- a/compiler/gen_llvm/src/llvm/convert.rs +++ b/compiler/gen_llvm/src/llvm/convert.rs @@ -27,6 +27,7 @@ pub fn basic_type_from_layout<'a, 'ctx, 'env>( match layout { Struct(sorted_fields) => basic_type_from_record(env, sorted_fields), + LambdaSet(lambda_set) => basic_type_from_layout(env, &lambda_set.runtime_representation()), Union(union_layout) => { use UnionLayout::*; @@ -96,7 +97,6 @@ pub fn basic_type_from_builtin<'a, 'ctx, 'env>( Float128 => context.f128_type().as_basic_type_enum(), Float64 => context.f64_type().as_basic_type_enum(), Float32 => context.f32_type().as_basic_type_enum(), - Float16 => context.f16_type().as_basic_type_enum(), Dict(_, _) | EmptyDict => zig_dict_type(env).into(), Set(_) | EmptySet => zig_dict_type(env).into(), List(_) | EmptyList => zig_list_type(env).into(), diff --git a/compiler/gen_llvm/src/llvm/refcounting.rs b/compiler/gen_llvm/src/llvm/refcounting.rs index 6a84b72167..8a89c924d7 100644 --- a/compiler/gen_llvm/src/llvm/refcounting.rs +++ b/compiler/gen_llvm/src/llvm/refcounting.rs @@ -626,6 +626,14 @@ fn modify_refcount_layout_build_function<'a, 'ctx, 'env>( Some(function) } }, + LambdaSet(lambda_set) => modify_refcount_layout_build_function( + env, + parent, + layout_ids, + mode, + when_recursive, + &lambda_set.runtime_representation(), + ), } } diff --git a/compiler/gen_wasm/README.md b/compiler/gen_wasm/README.md index 3e453b21e2..9681e350c3 100644 --- a/compiler/gen_wasm/README.md +++ b/compiler/gen_wasm/README.md @@ -3,42 +3,57 @@ ## Plan - Initial bringup - - Get a wasm backend working for some of the number tests. - - Use a separate `gen_wasm` directory for now, to avoid trying to do bringup and integration at the same time. + - [x] Get a wasm backend working for some of the number tests. + - [x] Use a separate `gen_wasm` directory for now, to avoid trying to do bringup and integration at the same time. +- Get the fundamentals working + + - [x] Come up with a way to do control flow + - [x] Flesh out the details of value representations between local variables and stack memory + - [x] Set up a way to write tests with any return value rather than just i64 and f64 + - [x] Implement stack memory + - [x] Push and pop stack frames + - [x] Deal with returning structs + - [x] Distinguish which variables go in locals, own stack frame, caller stack frame, etc. + - [ ] Ensure early Return statements don't skip stack cleanup + - [ ] Vendor-in parity_wasm library so that we can use `bumpalo::Vec` + - [ ] Implement relocations + - Requires knowing the _byte_ offset of each call site. This is awkward as the backend builds a `Vec` rather than a `Vec`. It may be worth serialising each instruction as it is inserted. + +- Refactor for code sharing with CPU backends + + - [ ] Implement a `scan_ast` pre-pass like `Backend` does, but for reusing Wasm locals rather than CPU registers + - [ ] Extract a trait from `WasmBackend` that looks as similar as possible to `Backend`, to prepare for code sharing + - [ ] Refactor to actually share code between `WasmBackend` and `Backend` if it seems feasible + - Integration - Move wasm files to `gen_dev/src/wasm` - Share tests between wasm and x64, with some way of saying which tests work on which backends, and dispatching to different eval helpers based on that. - Get `build_module` in object_builder.rs to dispatch to the wasm generator (adding some Wasm options to the `Triple` struct) - Get `build_module` to write to a file, or maybe return `Vec`, instead of returning an Object structure -- Code sharing - - Try to ensure that both Wasm and x64 use the same `Backend` trait so that we can share code. - - We need to work towards this after we've progressed a bit more with Wasm and gained more understanding and experience of the differences. - - We will have to think about how to deal with the `Backend` code that doesn't apply to Wasm. Perhaps we will end up with more traits like `RegisterBackend` / `StackBackend` or `NativeBackend` / `WasmBackend`, and perhaps even some traits to do with backends that support jumps and those that don't. ## Structured control flow -🚨 **This is an area that could be tricky** 🚨 - One of the security features of WebAssembly is that it does not allow unrestricted "jumps" to anywhere you like. It does not have an instruction for that. All of the [control instructions][control-inst] can only implement "structured" control flow, and have names like `if`, `loop`, `block` that you'd normally associate with high-level languages. There are branch (`br`) instructions that can jump to labelled blocks within the same function, but the blocks have to be nested in sensible ways. [control-inst]: https://webassembly.github.io/spec/core/syntax/instructions.html#control-instructions -Implications: +This way of representing control flow is similar to parts of the Roc AST like `When`, `If` and `LetRec`. But Mono IR converts this to jumps and join points, which are more of a Control Flow Graph than a tree. We need to map back from graph to a tree again in the Wasm backend. -Roc, like most modern languages, is already enforcing structured control flow in the source program. Constructs from the Roc AST like `When`, `If` and `LetRec` can all be converted straightforwardly to Wasm constructs. +Our solution is to wrap all joinpoint/jump graphs in an outer `loop`, with nested `block`s inside it. -However the Mono IR converts this to jumps and join points, which are more of a Control Flow Graph than a tree. That doesn't map so directly to the Wasm structures. This is such a common issue for compiler back-ends that the WebAssembly compiler toolkit `binaryen` has an [API for control-flow graphs][cfg-api]. We're not using `binaryen` right now. It's a C++ library, though it does have a (very thin and somewhat hard-to-use) [Rust wrapper][binaryen-rs]. We should probably investigate this area sooner rather than later. If relooping turns out to be necessary or difficult, we might need to switch from parity_wasm to binaryen. +### Possible future optimisations -> By the way, it's not obvious how to pronounce "binaryen" but apparently it rhymes with "Targaryen", the family name from the "Game of Thrones" TV series +There are other algorithms available that may result in more optimised control flow. We are not focusing on that for our development backend, but here are some notes for future reference. + +The WebAssembly compiler toolkit `binaryen` has an [API for control-flow graphs][cfg-api]. We're not using `binaryen` right now. It's a C++ library, though it does have a (very thin and somewhat hard-to-use) [Rust wrapper][binaryen-rs]. Binaryen's control-flow graph API implements the "Relooper" algorithm developed by the Emscripten project and described in [this paper](https://github.com/emscripten-core/emscripten/blob/main/docs/paper.pdf). + +> By the way, apparently "binaryen" rhymes with "Targaryen", the family name from the "Game of Thrones" TV series + +There is also an improvement on Relooper called ["Stackifier"](https://medium.com/leaningtech/solving-the-structured-control-flow-problem-once-and-for-all-5123117b1ee2). It can reorder the joinpoints and jumps to make code more efficient. (It is also has things Roc wouldn't need but C++ does, like support for "irreducible" graphs that include `goto`). [cfg-api]: https://github.com/WebAssembly/binaryen/wiki/Compiling-to-WebAssembly-with-Binaryen#cfg-api [binaryen-rs]: https://crates.io/crates/binaryen -Binaryen's control-flow graph API implements the "Relooper" algorithm developed by the Emscripten project and described in [this paper](https://github.com/emscripten-core/emscripten/blob/main/docs/paper.pdf). - -There is an alternative algorithm that is supposed to be an improvement on Relooper, called ["Stackifier"](https://medium.com/leaningtech/solving-the-structured-control-flow-problem-once-and-for-all-5123117b1ee2). - - ## Stack machine vs register machine Wasm's instruction set is based on a stack-machine VM. Whereas CPU instructions have named registers that they operate on, Wasm has no named registers at all. The instructions don't contain register names. Instructions can oly operate on whatever data is at the top of the stack. @@ -84,29 +99,30 @@ The Mono IR contains two functions, `Num.add` and `main`, so we generate two cor (func (;1;) (result i64) ; declare function index 1 (main) with no parameters and an i64 result (local i64 i64 i64 i64) ; declare 4 local variables, all with type i64, one for each symbol in the Mono IR - i64.const 1 ; load constant of type i64 and value 1 stack=[1] - local.set 0 ; store top of stack to local0 stack=[] local0=1 - i64.const 2 ; load constant of type i64 and value 2 stack=[2] local0=1 - local.set 1 ; store top of stack to local1 stack=[] local0=1 local1=2 - local.get 0 ; load local0 to top of stack stack=[1] local0=1 local1=2 - local.get 1 ; load local1 to top of stack stack=[1,2] local0=1 local1=2 - call 0 ; call function index 0 (which pops 2 and pushes 1) stack=[3] local0=1 local1=2 - local.set 2 ; store top of stack to local2 stack=[] local0=1 local1=2 local2=3 - i64.const 4 ; load constant of type i64 and value 4 stack=[4] local0=1 local1=2 local2=3 - local.set 3 ; store top of stack to local3 stack=[] local0=1 local1=2 local2=3 local3=4 - local.get 2 ; load local2 to top of stack stack=[3] local0=1 local1=2 local2=3 local3=4 - local.get 3 ; load local3 to top of stack stack=[3,4] local0=1 local1=2 local2=3 local3=4 - call 0 ; call function index 0 (which pops 2 and pushes 1) stack=[7] local0=1 local1=2 local2=3 local3=4 + i64.const 1 ; stack=[1] + local.set 0 ; stack=[] local0=1 + i64.const 2 ; stack=[2] local0=1 + local.set 1 ; stack=[] local0=1 local1=2 + local.get 0 ; stack=[1] local0=1 local1=2 + local.get 1 ; stack=[1,2] local0=1 local1=2 + call 0 ; stack=[3] local0=1 local1=2 + local.set 2 ; stack=[] local0=1 local1=2 local2=3 + i64.const 4 ; stack=[4] local0=1 local1=2 local2=3 + local.set 3 ; stack=[] local0=1 local1=2 local2=3 local3=4 + local.get 2 ; stack=[3] local0=1 local1=2 local2=3 local3=4 + local.get 3 ; stack=[3,4] local0=1 local1=2 local2=3 local3=4 + call 0 ; stack=[7] local0=1 local1=2 local2=3 local3=4 return) ; return the value at the top of the stack ``` -If we run this code through the `wasm-opt` tool from the [binaryen toolkit](https://github.com/WebAssembly/binaryen#tools), the unnecessary locals get optimised away. The command line below runs the minimum number of passes to achieve this (`--simplify-locals` must come first). +If we run this code through the `wasm-opt` tool from the [binaryen toolkit](https://github.com/WebAssembly/binaryen#tools), the unnecessary locals get optimised away (which is all of them in this example!). The command line below runs the minimum number of passes to achieve this (`--simplify-locals` must come first). ``` $ wasm-opt --simplify-locals --reorder-locals --vacuum example.wasm > opt.wasm ``` -The optimised functions have no local variables, and the code shrinks to about 60% of its original size. +The optimised functions have no local variables at all for this example. (Of course, this is an oversimplified toy example! It might not be so extreme in a real program.) + ``` (func (;0;) (param i64 i64) (result i64) local.get 0 @@ -116,9 +132,20 @@ The optimised functions have no local variables, and the code shrinks to about 6 i64.const 1 i64.const 2 call 0 - i64.const 4) + i64.const 4 + call 0) ``` +### Reducing sets and gets + +It would be nice to find some cheap optimisation to reduce the number of `local.set` and `local.get` instructions. + +We don't need a `local` if the value we want is already at the top of the VM stack. In fact, for our example above, it just so happens that if we simply skip generating the `local.set` instructions, everything _does_ appear on the VM stack in the right order, which means we can skip the `local.get` too. It ends up being very close to the fully optimised version! I assume this is because the Mono IR within the function is in dependency order, but I'm not sure... + +Of course the trick is to do this reliably for more complex dependency graphs. I am investigating whether we can do it by optimistically assuming it's OK not to create a local, and then keeping track of which symbols are at which positions in the VM stack after every instruction. Then when we need to use a symbol we can first check if it's on the VM stack and only create a local if it's not. In cases where we _do_ need to create a local, we need to go back and insert a `local.set` instruction at an earlier point in the program. We can make this fast by waiting to do all of the insertions in one batch when we're finalising the procedure. + +For a while we thought it would be very helpful to reuse the same local for multiple symbols at different points in the program. And we already have similar code in the CPU backends for register allocation. But on further examination, it doesn't actually buy us much! In our example above, we would still have the same number of `local.set` and `local.get` instructions - they'd just be operating on two locals instead of four! That doesn't shrink much code. Only the declaration at the top of the function would shrink from `(local i64 i64 i64 i64)` to `(local i64 i64)`... and in fact that's only smaller in the text format, it's the same size in the binary format! So the `scan_ast` pass doesn't seem worthwhile for Wasm. + ## Memory WebAssembly programs have a "linear memory" for storing data, which is a block of memory assigned to it by the host. You can assign a min and max size to the memory, and the WebAssembly program can request 64kB pages from the host, just like a "normal" program would request pages from the OS. Addresses start at zero and go up to whatever the current size is. Zero is a perfectly normal address like any other, and dereferencing it is not a segfault. But addresses beyond the current memory size are out of bounds and dereferencing them will cause a panic. @@ -137,7 +164,7 @@ When we are talking about how we store values in _memory_, I'll use the term _st Of course our program can use another area of memory as a heap as well. WebAssembly doesn't mind how you divide up your memory. It just gives you some memory and some instructions for loading and storing. -## Function calls +## Calling conventions & stack memory In WebAssembly you call a function by pushing arguments to the stack and then issuing a `call` instruction, which specifies a function index. The VM knows how many values to pop off the stack by examining the _type_ of the function. In our example earlier, `Num.add` had the type `[i64 i64] → [i64]` so it expects to find two i64's on the stack and pushes one i64 back as the result. Remember, the runtime engine will validate the module before running it, and if your generated code is trying to call a function at a point in the program where the wrong value types are on the stack, it will fail validation. @@ -145,11 +172,17 @@ Function arguments are restricted to the four value types, `i32`, `i64`, `f32` a That's all great for primitive values but what happens when we want to pass more complex data structures between functions? -Well, remember, "stack memory" is not a special kind of memory in WebAssembly, it's just an area of our memory where we _decide_ that we want to implement a stack data structure. So we can implement it however we want. A good choice would be to make our stack frame look the same as it would when we're targeting a CPU, except without the return address (since there's no need for one). We can also decide to pass numbers through the machine stack rather than in stack memory, since that takes fewer instructions. +Well, remember, "stack memory" is not a special kind of memory in WebAssembly, and is separate from the VM stack. It's just an area of our memory where we implement a stack data structure. But there are some conventions that it makes sense to follow so that we can easily link to Wasm code generated from Zig or other languages. -The only other thing we need is a stack pointer. On CPU targets, there's often have a specific "stack pointer" register. WebAssembly has no equivalent to that, but we can use a `global` variable. +### Observations from compiled C code -The system I've outlined above is based on my experience of compiling C to WebAssembly via the Emscripten toolchain (which is built on top of clang). It's also in line with what the WebAssembly project describes [here](https://github.com/WebAssembly/design/blob/main/Rationale.md#locals). +- `global 0` is used as the stack pointer, and its value is normally copied to a `local` as well (presumably because locals tend to be assigned to CPU registers) +- Stack memory grows downwards +- If a C function returns a struct, the compiled WebAssembly function has no return value, but instead has an extra _argument_. The argument is an `i32` pointer to space allocated in the caller's stack, that the called function can write to. +- There is no maximum number of arguments for a WebAssembly function, and arguments are not passed via _stack memory_. This makes sense because the _VM stack_ has no size limit. It's like having a CPU with an unlimited number of registers. +- Stack memory is only used for allocating local variables, not for passing arguments. And it's only used for values that cannot be stored in one of WebAssembly's primitive values (`i32`, `i64`, `f32`, `f64`). + +These observations are based on experiments compiling C to WebAssembly via the Emscripten toolchain (which is built on top of clang). It's also in line with what the WebAssembly project describes [here](https://github.com/WebAssembly/design/blob/main/Rationale.md#locals). ## Modules vs Instances diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index 270ddb4e6a..303fe38349 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -1,5 +1,5 @@ use parity_wasm::builder; -use parity_wasm::builder::{CodeLocation, ModuleBuilder}; +use parity_wasm::builder::{CodeLocation, FunctionDefinition, ModuleBuilder, SignatureBuilder}; use parity_wasm::elements::{ BlockType, Instruction, Instruction::*, Instructions, Local, ValueType, }; @@ -10,47 +10,26 @@ use roc_module::symbol::Symbol; use roc_mono::ir::{CallType, Expr, JoinPointId, Literal, Proc, Stmt}; use roc_mono::layout::{Builtin, Layout}; +use crate::layout::WasmLayout; +use crate::storage::SymbolStorage; +use crate::{ + copy_memory, pop_stack_frame, push_stack_frame, round_up_to_alignment, LocalId, PTR_SIZE, + PTR_TYPE, +}; + // Don't allocate any constant data at address zero or near it. Would be valid, but bug-prone. // Follow Emscripten's example by using 1kB (4 bytes would probably do) const UNUSED_DATA_SECTION_BYTES: u32 = 1024; -#[derive(Clone, Copy, Debug)] -struct LocalId(u32); - #[derive(Clone, Copy, Debug)] struct LabelId(u32); -#[derive(Debug)] -struct SymbolStorage(LocalId, WasmLayout); - -#[derive(Clone, Copy, Debug)] -struct WasmLayout { - value_type: ValueType, - stack_memory: u32, -} - -impl WasmLayout { - fn new(layout: &Layout) -> Result { - match layout { - Layout::Builtin(Builtin::Int1 | Builtin::Int8 | Builtin::Int16 | Builtin::Int32) => { - Ok(Self { - value_type: ValueType::I32, - stack_memory: 0, - }) - } - Layout::Builtin(Builtin::Int64) => Ok(Self { - value_type: ValueType::I64, - stack_memory: 0, - }), - Layout::Builtin(Builtin::Float64) => Ok(Self { - value_type: ValueType::F64, - stack_memory: 0, - }), - x => Err(format!("layout, {:?}, not implemented yet", x)), - } - } +enum LocalKind { + Parameter, + Variable, } +// TODO: use Bumpalo Vec once parity_wasm supports general iterators (>=0.43) pub struct WasmBackend<'a> { // Module: Wasm AST pub builder: ModuleBuilder, @@ -62,12 +41,12 @@ pub struct WasmBackend<'a> { // Functions: Wasm AST instructions: std::vec::Vec, - ret_type: ValueType, arg_types: std::vec::Vec, locals: std::vec::Vec, // Functions: internal state & IR mappings - stack_memory: u32, + stack_memory: i32, + stack_frame_pointer: Option, symbol_storage_map: MutMap, /// how many blocks deep are we (used for jumps) block_depth: u32, @@ -87,12 +66,12 @@ impl<'a> WasmBackend<'a> { // Functions: Wasm AST instructions: std::vec::Vec::with_capacity(256), - ret_type: ValueType::I32, arg_types: std::vec::Vec::with_capacity(8), locals: std::vec::Vec::with_capacity(32), // Functions: internal state & IR mappings stack_memory: 0, + stack_frame_pointer: None, symbol_storage_map: MutMap::default(), block_depth: 0, joinpoint_label_map: MutMap::default(), @@ -107,48 +86,18 @@ impl<'a> WasmBackend<'a> { // Functions: internal state & IR mappings self.stack_memory = 0; + self.stack_frame_pointer = None; self.symbol_storage_map.clear(); - // joinpoint_label_map.clear(); + self.joinpoint_label_map.clear(); + assert_eq!(self.block_depth, 0); } pub fn build_proc(&mut self, proc: Proc<'a>, sym: Symbol) -> Result { - let ret_layout = WasmLayout::new(&proc.ret_layout)?; - if ret_layout.stack_memory > 0 { - // TODO: if returning a struct by value, add an extra argument for a pointer to callee's stack memory - return Err(format!( - "Not yet implemented: Return in stack memory for non-primtitive layouts like {:?}", - proc.ret_layout - )); - } - - self.ret_type = ret_layout.value_type; - self.arg_types.reserve(proc.args.len()); - - for (layout, symbol) in proc.args { - let wasm_layout = WasmLayout::new(layout)?; - self.arg_types.push(wasm_layout.value_type); - self.insert_local(wasm_layout, *symbol); - } + let signature_builder = self.start_proc(&proc); self.build_stmt(&proc.body, &proc.ret_layout)?; - let signature = builder::signature() - .with_params(self.arg_types.clone()) // requires std::Vec, not Bumpalo - .with_result(self.ret_type) - .build_sig(); - - // functions must end with an End instruction/opcode - let mut instructions = self.instructions.clone(); - instructions.push(Instruction::End); - - let function_def = builder::function() - .with_signature(signature) - .body() - .with_locals(self.locals.clone()) - .with_instructions(Instructions::new(instructions)) - .build() // body - .build(); // function - + let function_def = self.finalize_proc(signature_builder); let location = self.builder.push_function(function_def); let function_index = location.body; self.proc_symbol_map.insert(sym, location); @@ -157,32 +106,169 @@ impl<'a> WasmBackend<'a> { Ok(function_index) } - fn insert_local(&mut self, layout: WasmLayout, symbol: Symbol) -> LocalId { - self.stack_memory += layout.stack_memory; - let index = self.symbol_storage_map.len(); - if index >= self.arg_types.len() { - self.locals.push(Local::new(1, layout.value_type)); + fn start_proc(&mut self, proc: &Proc<'a>) -> SignatureBuilder { + let ret_layout = WasmLayout::new(&proc.ret_layout); + + let signature_builder = if let WasmLayout::StackMemory { .. } = ret_layout { + self.arg_types.push(PTR_TYPE); + self.start_block(BlockType::NoResult); // block to ensure all paths pop stack memory (if any) + builder::signature() + } else { + let ret_type = ret_layout.value_type(); + self.start_block(BlockType::Value(ret_type)); // block to ensure all paths pop stack memory (if any) + builder::signature().with_result(ret_type) + }; + + for (layout, symbol) in proc.args { + self.insert_local(WasmLayout::new(layout), *symbol, LocalKind::Parameter); } - let local_id = LocalId(index as u32); - let storage = SymbolStorage(local_id, layout); - self.symbol_storage_map.insert(symbol, storage); - local_id + + signature_builder.with_params(self.arg_types.clone()) } - fn get_symbol_storage(&self, sym: &Symbol) -> Result<&SymbolStorage, String> { - self.symbol_storage_map.get(sym).ok_or_else(|| { - format!( + fn finalize_proc(&mut self, signature_builder: SignatureBuilder) -> FunctionDefinition { + self.end_block(); // end the block from start_proc, to ensure all paths pop stack memory (if any) + + let mut final_instructions = Vec::with_capacity(self.instructions.len() + 10); + + if self.stack_memory > 0 { + push_stack_frame( + &mut final_instructions, + self.stack_memory, + self.stack_frame_pointer.unwrap(), + ); + } + + final_instructions.extend(self.instructions.drain(0..)); + + if self.stack_memory > 0 { + pop_stack_frame( + &mut final_instructions, + self.stack_memory, + self.stack_frame_pointer.unwrap(), + ); + } + final_instructions.push(End); + + builder::function() + .with_signature(signature_builder.build_sig()) + .body() + .with_locals(self.locals.clone()) + .with_instructions(Instructions::new(final_instructions)) + .build() // body + .build() // function + } + + fn insert_local( + &mut self, + wasm_layout: WasmLayout, + symbol: Symbol, + kind: LocalKind, + ) -> SymbolStorage { + let local_id = LocalId((self.arg_types.len() + self.locals.len()) as u32); + + let storage = match kind { + LocalKind::Parameter => { + // Already stack-allocated by the caller if needed. + self.arg_types.push(wasm_layout.value_type()); + match wasm_layout { + WasmLayout::LocalOnly(value_type, size) => SymbolStorage::ParamPrimitive { + local_id, + value_type, + size, + }, + + WasmLayout::HeapMemory => SymbolStorage::ParamPrimitive { + local_id, + value_type: PTR_TYPE, + size: PTR_SIZE, + }, + + WasmLayout::StackMemory { + size, + alignment_bytes, + } => SymbolStorage::ParamStackMemory { + local_id, + size, + alignment_bytes, + }, + } + } + LocalKind::Variable => { + self.locals.push(Local::new(1, wasm_layout.value_type())); + + match wasm_layout { + WasmLayout::LocalOnly(value_type, size) => SymbolStorage::VarPrimitive { + local_id, + value_type, + size, + }, + + WasmLayout::HeapMemory => SymbolStorage::VarHeapMemory { local_id }, + + WasmLayout::StackMemory { + size, + alignment_bytes, + } => { + let offset = + round_up_to_alignment(self.stack_memory, alignment_bytes as i32); + self.stack_memory = offset + size as i32; + + match self.stack_frame_pointer { + None => { + // This is the first stack-memory variable in the function + // That means we can reuse it as the stack frame pointer, + // and it will get initialised at the start of the function + self.stack_frame_pointer = Some(local_id); + } + + Some(frame_ptr_id) => { + // This local points to the base of a struct, at an offset from the stack frame pointer + // Having one local per variable means params and locals work the same way in code gen. + // (alternatively we could use one frame pointer + offset for all struct variables) + self.instructions.extend([ + GetLocal(frame_ptr_id.0), + I32Const(offset), + I32Add, + SetLocal(local_id.0), + ]); + } + }; + + SymbolStorage::VarStackMemory { + local_id, + size, + offset: offset as u32, + alignment_bytes, + } + } + } + } + }; + + self.symbol_storage_map.insert(symbol, storage.clone()); + + storage + } + + fn get_symbol_storage(&self, sym: &Symbol) -> &SymbolStorage { + self.symbol_storage_map.get(sym).unwrap_or_else(|| { + panic!( "Symbol {:?} not found in function scope:\n{:?}", sym, self.symbol_storage_map ) }) } - fn load_from_symbol(&mut self, sym: &Symbol) -> Result<(), String> { - let SymbolStorage(LocalId(local_id), _) = self.get_symbol_storage(sym)?; - let id: u32 = *local_id; - self.instructions.push(GetLocal(id)); - Ok(()) + fn local_id_from_symbol(&self, sym: &Symbol) -> LocalId { + let storage = self.get_symbol_storage(sym); + storage.local_id() + } + + fn load_symbol(&mut self, sym: &Symbol) { + let storage = self.get_symbol_storage(sym); + let index: u32 = storage.local_id().0; + self.instructions.push(GetLocal(index)); } /// start a loop that leaves a value on the stack @@ -193,12 +279,9 @@ impl<'a> WasmBackend<'a> { self.instructions.push(Loop(BlockType::Value(value_type))); } - fn start_block(&mut self) { + fn start_block(&mut self, block_type: BlockType) { self.block_depth += 1; - - // Our blocks always end with a `return` or `br`, - // so they never leave extra values on the stack - self.instructions.push(Block(BlockType::NoResult)); + self.instructions.push(Block(block_type)); } fn end_block(&mut self) { @@ -208,36 +291,77 @@ impl<'a> WasmBackend<'a> { fn build_stmt(&mut self, stmt: &Stmt<'a>, ret_layout: &Layout<'a>) -> Result<(), String> { match stmt { - // This pattern is a simple optimisation to get rid of one local and two instructions per proc. - // If we are just returning the expression result, then don't SetLocal and immediately GetLocal + // Simple optimisation: if we are just returning the expression, we don't need a local Stmt::Let(let_sym, expr, layout, Stmt::Ret(ret_sym)) if let_sym == ret_sym => { + let wasm_layout = WasmLayout::new(layout); + if let WasmLayout::StackMemory { + size, + alignment_bytes, + } = wasm_layout + { + // Map this symbol to the first argument (pointer into caller's stack) + // Saves us from having to copy it later + let storage = SymbolStorage::ParamStackMemory { + local_id: LocalId(0), + size, + alignment_bytes, + }; + self.symbol_storage_map.insert(*let_sym, storage); + } self.build_expr(let_sym, expr, layout)?; - self.instructions.push(Return); + self.instructions.push(Br(self.block_depth)); // jump to end of function (stack frame pop) Ok(()) } Stmt::Let(sym, expr, layout, following) => { - let wasm_layout = WasmLayout::new(layout)?; - let local_id = self.insert_local(wasm_layout, *sym); + let wasm_layout = WasmLayout::new(layout); + let local_id = self + .insert_local(wasm_layout, *sym, LocalKind::Variable) + .local_id(); self.build_expr(sym, expr, layout)?; - self.instructions.push(SetLocal(local_id.0)); + + // If this local is shared with the stack frame pointer, it's already assigned + match self.stack_frame_pointer { + Some(sfp) if sfp == local_id => {} + _ => self.instructions.push(SetLocal(local_id.0)), + } self.build_stmt(following, ret_layout)?; Ok(()) } Stmt::Ret(sym) => { - if let Some(SymbolStorage(local_id, _)) = self.symbol_storage_map.get(sym) { - self.instructions.push(GetLocal(local_id.0)); - self.instructions.push(Return); - Ok(()) - } else { - Err(format!( - "Not yet implemented: returning values with layout {:?}", - ret_layout - )) + use crate::storage::SymbolStorage::*; + + let storage = self.symbol_storage_map.get(sym).unwrap(); + + match storage { + VarStackMemory { + local_id, + size, + alignment_bytes, + .. + } + | ParamStackMemory { + local_id, + size, + alignment_bytes, + } => { + let from = *local_id; + let to = LocalId(0); + copy_memory(&mut self.instructions, from, to, *size, *alignment_bytes, 0); + } + + ParamPrimitive { local_id, .. } + | VarPrimitive { local_id, .. } + | VarHeapMemory { local_id, .. } => { + self.instructions.push(GetLocal(local_id.0)); + self.instructions.push(Br(self.block_depth)); // jump to end of function (for stack frame pop) + } } + + Ok(()) } Stmt::Switch { @@ -253,19 +377,16 @@ impl<'a> WasmBackend<'a> { // create (number_of_branches - 1) new blocks. for _ in 0..branches.len() { - self.start_block() + self.start_block(BlockType::NoResult) } // the LocalId of the symbol that we match on - let matched_on = match self.symbol_storage_map.get(cond_symbol) { - Some(SymbolStorage(local_id, _)) => local_id.0, - None => unreachable!("symbol not defined: {:?}", cond_symbol), - }; + let matched_on = self.local_id_from_symbol(cond_symbol); // then, we jump whenever the value under scrutiny is equal to the value of a branch for (i, (value, _, _)) in branches.iter().enumerate() { // put the cond_symbol on the top of the stack - self.instructions.push(GetLocal(matched_on)); + self.instructions.push(GetLocal(matched_on.0)); self.instructions.push(I32Const(*value as i32)); @@ -299,13 +420,15 @@ impl<'a> WasmBackend<'a> { // make locals for join pointer parameters let mut jp_parameter_local_ids = std::vec::Vec::with_capacity(parameters.len()); for parameter in parameters.iter() { - let wasm_layout = WasmLayout::new(¶meter.layout)?; - let local_id = self.insert_local(wasm_layout, parameter.symbol); + let wasm_layout = WasmLayout::new(¶meter.layout); + let local_id = self + .insert_local(wasm_layout, parameter.symbol, LocalKind::Variable) + .local_id(); jp_parameter_local_ids.push(local_id); } - self.start_block(); + self.start_block(BlockType::NoResult); self.joinpoint_label_map .insert(*id, (self.block_depth, jp_parameter_local_ids)); @@ -316,8 +439,8 @@ impl<'a> WasmBackend<'a> { // A `return` inside of a `loop` seems to make it so that the `loop` itself // also "returns" (so, leaves on the stack) a value of the return type. - let return_wasm_layout = WasmLayout::new(ret_layout)?; - self.start_loop_with_return(return_wasm_layout.value_type); + let return_wasm_layout = WasmLayout::new(ret_layout); + self.start_loop_with_return(return_wasm_layout.value_type()); self.build_stmt(body, ret_layout)?; @@ -331,12 +454,8 @@ impl<'a> WasmBackend<'a> { // put the arguments on the stack for (symbol, local_id) in arguments.iter().zip(locals.iter()) { - let argument = match self.symbol_storage_map.get(symbol) { - Some(SymbolStorage(local_id, _)) => local_id.0, - None => unreachable!("symbol not defined: {:?}", symbol), - }; - - self.instructions.push(GetLocal(argument)); + let argument = self.local_id_from_symbol(symbol); + self.instructions.push(GetLocal(argument.0)); self.instructions.push(SetLocal(local_id.0)); } @@ -365,7 +484,7 @@ impl<'a> WasmBackend<'a> { }) => match call_type { CallType::ByName { name: func_sym, .. } => { for arg in *arguments { - self.load_from_symbol(arg)?; + self.load_symbol(arg); } let function_location = self.proc_symbol_map.get(func_sym).ok_or(format!( "Cannot find function {:?} called from {:?}", @@ -381,38 +500,112 @@ impl<'a> WasmBackend<'a> { x => Err(format!("the call type, {:?}, is not yet implemented", x)), }, + Expr::Struct(fields) => self.create_struct(sym, layout, fields), + x => Err(format!("Expression is not yet implemented {:?}", x)), } } fn load_literal(&mut self, lit: &Literal<'a>, layout: &Layout<'a>) -> Result<(), String> { - match lit { - Literal::Bool(x) => { - self.instructions.push(I32Const(*x as i32)); - Ok(()) - } - Literal::Byte(x) => { - self.instructions.push(I32Const(*x as i32)); - Ok(()) - } - Literal::Int(x) => { - match layout { - Layout::Builtin(Builtin::Int32) => { - self.instructions.push(I32Const(*x as i32)); - } - Layout::Builtin(Builtin::Int64) => { - self.instructions.push(I64Const(*x as i64)); - } - x => panic!("loading literal, {:?}, is not yet implemented", x), + let instruction = match lit { + Literal::Bool(x) => I32Const(*x as i32), + Literal::Byte(x) => I32Const(*x as i32), + Literal::Int(x) => match layout { + Layout::Builtin(Builtin::Int64) => I64Const(*x as i64), + Layout::Builtin( + Builtin::Int32 + | Builtin::Int16 + | Builtin::Int8 + | Builtin::Int1 + | Builtin::Usize, + ) => I32Const(*x as i32), + x => { + return Err(format!("loading literal, {:?}, is not yet implemented", x)); } - Ok(()) + }, + Literal::Float(x) => match layout { + Layout::Builtin(Builtin::Float64) => F64Const((*x as f64).to_bits()), + Layout::Builtin(Builtin::Float32) => F32Const((*x as f32).to_bits()), + x => { + return Err(format!("loading literal, {:?}, is not yet implemented", x)); + } + }, + x => { + return Err(format!("loading literal, {:?}, is not yet implemented", x)); } - Literal::Float(x) => { - let val: f64 = *x; - self.instructions.push(F64Const(val.to_bits())); - Ok(()) + }; + self.instructions.push(instruction); + Ok(()) + } + + fn create_struct( + &mut self, + sym: &Symbol, + layout: &Layout<'a>, + fields: &'a [Symbol], + ) -> Result<(), String> { + let storage = self.get_symbol_storage(sym).to_owned(); + + if let Layout::Struct(field_layouts) = layout { + match storage { + SymbolStorage::VarStackMemory { local_id, size, .. } + | SymbolStorage::ParamStackMemory { local_id, size, .. } => { + if size > 0 { + let mut relative_offset = 0; + for (field, _) in fields.iter().zip(field_layouts.iter()) { + relative_offset += self.copy_symbol_to_pointer_at_offset( + local_id, + relative_offset, + field, + ); + } + } else { + return Err(format!("Not supported yet: zero-size struct at {:?}", sym)); + } + } + _ => { + return Err(format!( + "Cannot create struct {:?} with storage {:?}", + sym, storage + )); + } } - x => Err(format!("loading literal, {:?}, is not yet implemented", x)), + } else { + // Struct expression but not Struct layout => single element. Copy it. + let field_storage = self.get_symbol_storage(&fields[0]).to_owned(); + self.copy_storage(&storage, &field_storage); + } + Ok(()) + } + + fn copy_symbol_to_pointer_at_offset( + &mut self, + to_ptr: LocalId, + to_offset: u32, + from_symbol: &Symbol, + ) -> u32 { + let from_storage = self.get_symbol_storage(from_symbol).to_owned(); + from_storage.copy_to_memory(&mut self.instructions, to_ptr, to_offset) + } + + fn copy_storage(&mut self, to: &SymbolStorage, from: &SymbolStorage) { + let has_stack_memory = to.has_stack_memory(); + debug_assert!(from.has_stack_memory() == has_stack_memory); + + if !has_stack_memory { + debug_assert!(from.value_type() == to.value_type()); + self.instructions.push(GetLocal(from.local_id().0)); + self.instructions.push(SetLocal(to.local_id().0)); + } else { + let (size, alignment_bytes) = from.stack_size_and_alignment(); + copy_memory( + &mut self.instructions, + from.local_id(), + to.local_id(), + size, + alignment_bytes, + 0, + ); } } @@ -423,10 +616,10 @@ impl<'a> WasmBackend<'a> { return_layout: &Layout<'a>, ) -> Result<(), String> { for arg in args { - self.load_from_symbol(arg)?; + self.load_symbol(arg); } - let wasm_layout = WasmLayout::new(return_layout)?; - self.build_instructions_lowlevel(lowlevel, wasm_layout.value_type)?; + let wasm_layout = WasmLayout::new(return_layout); + self.build_instructions_lowlevel(lowlevel, wasm_layout.value_type())?; Ok(()) } @@ -441,7 +634,7 @@ impl<'a> WasmBackend<'a> { // For those, we'll need to pre-process each argument before the main op, // so simple arrays of instructions won't work. But there are common patterns. let instructions: &[Instruction] = match lowlevel { - // Wasm type might not be enough, may need to sign-extend i8 etc. Maybe in load_from_symbol? + // Wasm type might not be enough, may need to sign-extend i8 etc. Maybe in load_symbol? LowLevel::NumAdd => match return_value_type { ValueType::I32 => &[I32Add], ValueType::I64 => &[I64Add], diff --git a/compiler/gen_wasm/src/layout.rs b/compiler/gen_wasm/src/layout.rs new file mode 100644 index 0000000000..df59b80eb1 --- /dev/null +++ b/compiler/gen_wasm/src/layout.rs @@ -0,0 +1,82 @@ +use parity_wasm::elements::ValueType; +use roc_mono::layout::{Layout, UnionLayout}; + +use crate::{PTR_SIZE, PTR_TYPE}; + +// See README for background information on Wasm locals, memory and function calls +#[derive(Debug, Clone)] +pub enum WasmLayout { + // Primitive number value. Just a Wasm local, without any stack memory. + // For example, Roc i8 is represented as Wasm i32. Store the type and the original size. + LocalOnly(ValueType, u32), + + // Local pointer to stack memory + StackMemory { size: u32, alignment_bytes: u32 }, + + // Local pointer to heap memory + HeapMemory, +} + +impl WasmLayout { + pub fn new(layout: &Layout) -> Self { + use roc_mono::layout::Builtin::*; + use UnionLayout::*; + use ValueType::*; + + let size = layout.stack_size(PTR_SIZE); + let alignment_bytes = layout.alignment_bytes(PTR_SIZE); + + match layout { + Layout::Builtin(Int32 | Int16 | Int8 | Int1 | Usize) => Self::LocalOnly(I32, size), + + Layout::Builtin(Int64) => Self::LocalOnly(I64, size), + + Layout::Builtin(Float32) => Self::LocalOnly(F32, size), + + Layout::Builtin(Float64) => Self::LocalOnly(F64, size), + + Layout::Builtin( + Int128 + | Decimal + | Float128 + | Str + | Dict(_, _) + | Set(_) + | List(_) + | EmptyStr + | EmptyList + | EmptyDict + | EmptySet, + ) + | Layout::Struct(_) + | Layout::LambdaSet(_) + | Layout::Union(NonRecursive(_)) => Self::StackMemory { + size, + alignment_bytes, + }, + + Layout::Union( + Recursive(_) + | NonNullableUnwrapped(_) + | NullableWrapped { .. } + | NullableUnwrapped { .. }, + ) + | Layout::RecursivePointer => Self::HeapMemory, + } + } + + pub fn value_type(&self) -> ValueType { + match self { + Self::LocalOnly(type_, _) => *type_, + _ => PTR_TYPE, + } + } + + #[allow(dead_code)] + pub fn stack_memory(&self) -> u32 { + match self { + Self::StackMemory { size, .. } => *size, + _ => 0, + } + } +} diff --git a/compiler/gen_wasm/src/lib.rs b/compiler/gen_wasm/src/lib.rs index e814a7a3b7..32ff3e83ad 100644 --- a/compiler/gen_wasm/src/lib.rs +++ b/compiler/gen_wasm/src/lib.rs @@ -1,9 +1,11 @@ mod backend; pub mod from_wasm32_memory; +mod layout; +mod storage; use bumpalo::Bump; use parity_wasm::builder; -use parity_wasm::elements::Internal; +use parity_wasm::elements::{Instruction, Instruction::*, Internal, ValueType}; use roc_collections::all::{MutMap, MutSet}; use roc_module::symbol::{Interns, Symbol}; @@ -12,6 +14,21 @@ use roc_mono::layout::LayoutIds; use crate::backend::WasmBackend; +const PTR_SIZE: u32 = 4; +const PTR_TYPE: ValueType = ValueType::I32; + +// All usages of these alignment constants take u32, so an enum wouldn't add any safety. +pub const ALIGN_1: u32 = 0; +pub const ALIGN_2: u32 = 1; +pub const ALIGN_4: u32 = 2; +pub const ALIGN_8: u32 = 3; + +pub const STACK_POINTER_GLOBAL_ID: u32 = 0; +pub const STACK_ALIGNMENT_BYTES: i32 = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LocalId(pub u32); + pub struct Env<'a> { pub arena: &'a Bump, // not really using this much, parity_wasm works with std::vec a lot pub interns: Interns, @@ -21,7 +38,18 @@ pub struct Env<'a> { pub fn build_module<'a>( env: &'a Env, procedures: MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>, -) -> Result, String> { +) -> Result, String> { + let (builder, _) = build_module_help(env, procedures)?; + let module = builder.build(); + module + .to_bytes() + .map_err(|e| -> String { format!("Error serialising Wasm module {:?}", e) }) +} + +pub fn build_module_help<'a>( + env: &'a Env, + procedures: MutMap<(Symbol, ProcLayout<'a>), Proc<'a>>, +) -> Result<(builder::ModuleBuilder, u32), String> { let mut backend = WasmBackend::new(); let mut layout_ids = LayoutIds::default(); @@ -38,8 +66,9 @@ pub fn build_module<'a>( let mut procedures: std::vec::Vec<_> = procedures.into_iter().collect(); procedures.sort_by(|a, b| b.0 .0.cmp(&a.0 .0)); + let mut function_index: u32 = 0; for ((sym, layout), proc) in procedures { - let function_index = backend.build_proc(proc, sym)?; + function_index = backend.build_proc(proc, sym)?; if env.exposed_to_host.contains(&sym) { let fn_name = layout_ids .get_toplevel(sym, &layout) @@ -54,6 +83,11 @@ pub fn build_module<'a>( } } + // Because of the sorting above, we know the last function in the `for` is the main function. + // Here we grab its index and return it, so that the test_wrapper is able to call it. + // This is a workaround until we implement object files with symbols and relocations. + let main_function_index = function_index; + const MIN_MEMORY_SIZE_KB: u32 = 1024; const PAGE_SIZE_KB: u32 = 64; @@ -61,15 +95,98 @@ pub fn build_module<'a>( .with_min(MIN_MEMORY_SIZE_KB / PAGE_SIZE_KB) .build(); backend.builder.push_memory(memory); - let memory_export = builder::export() .field("memory") .with_internal(Internal::Memory(0)) .build(); backend.builder.push_export(memory_export); - let module = backend.builder.build(); - module - .to_bytes() - .map_err(|e| -> String { format!("Error serialising Wasm module {:?}", e) }) + let stack_pointer_global = builder::global() + .with_type(PTR_TYPE) + .mutable() + .init_expr(Instruction::I32Const((MIN_MEMORY_SIZE_KB * 1024) as i32)) + .build(); + backend.builder.push_global(stack_pointer_global); + + Ok((backend.builder, main_function_index)) +} + +fn encode_alignment(bytes: u32) -> u32 { + match bytes { + 1 => ALIGN_1, + 2 => ALIGN_2, + 4 => ALIGN_4, + 8 => ALIGN_8, + _ => panic!("{:?}-byte alignment is not supported", bytes), + } +} + +fn copy_memory( + instructions: &mut Vec, + from_ptr: LocalId, + to_ptr: LocalId, + size: u32, + alignment_bytes: u32, + offset: u32, +) { + let alignment_flag = encode_alignment(alignment_bytes); + let mut current_offset = offset; + while size - current_offset >= 8 { + instructions.push(GetLocal(to_ptr.0)); + instructions.push(GetLocal(from_ptr.0)); + instructions.push(I64Load(alignment_flag, current_offset)); + instructions.push(I64Store(alignment_flag, current_offset)); + current_offset += 8; + } + if size - current_offset >= 4 { + instructions.push(GetLocal(to_ptr.0)); + instructions.push(GetLocal(from_ptr.0)); + instructions.push(I32Load(alignment_flag, current_offset)); + instructions.push(I32Store(alignment_flag, current_offset)); + current_offset += 4; + } + while size - current_offset > 0 { + instructions.push(GetLocal(to_ptr.0)); + instructions.push(GetLocal(from_ptr.0)); + instructions.push(I32Load8U(alignment_flag, current_offset)); + instructions.push(I32Store8(alignment_flag, current_offset)); + current_offset += 1; + } +} + +/// Round up to alignment_bytes (assumed to be a power of 2) +pub fn round_up_to_alignment(unaligned: i32, alignment_bytes: i32) -> i32 { + let mut aligned = unaligned; + aligned += alignment_bytes - 1; // if lower bits are non-zero, push it over the next boundary + aligned &= -alignment_bytes; // mask with a flag that has upper bits 1, lower bits 0 + aligned +} + +pub fn push_stack_frame( + instructions: &mut Vec, + size: i32, + local_frame_pointer: LocalId, +) { + let aligned_size = round_up_to_alignment(size, STACK_ALIGNMENT_BYTES); + instructions.extend([ + GetGlobal(STACK_POINTER_GLOBAL_ID), + I32Const(aligned_size), + I32Sub, + TeeLocal(local_frame_pointer.0), + SetGlobal(STACK_POINTER_GLOBAL_ID), + ]); +} + +pub fn pop_stack_frame( + instructions: &mut Vec, + size: i32, + local_frame_pointer: LocalId, +) { + let aligned_size = round_up_to_alignment(size, STACK_ALIGNMENT_BYTES); + instructions.extend([ + GetLocal(local_frame_pointer.0), + I32Const(aligned_size), + I32Add, + SetGlobal(STACK_POINTER_GLOBAL_ID), + ]); } diff --git a/compiler/gen_wasm/src/storage.rs b/compiler/gen_wasm/src/storage.rs new file mode 100644 index 0000000000..51c14203b1 --- /dev/null +++ b/compiler/gen_wasm/src/storage.rs @@ -0,0 +1,146 @@ +use crate::{copy_memory, LocalId, ALIGN_1, ALIGN_2, ALIGN_4, ALIGN_8}; +use parity_wasm::elements::{Instruction, Instruction::*, ValueType}; + +#[derive(Debug, Clone)] +pub enum SymbolStorage { + VarPrimitive { + local_id: LocalId, + value_type: ValueType, + size: u32, + }, + ParamPrimitive { + local_id: LocalId, + value_type: ValueType, + size: u32, + }, + VarStackMemory { + local_id: LocalId, + size: u32, + offset: u32, + alignment_bytes: u32, + }, + ParamStackMemory { + local_id: LocalId, + size: u32, + alignment_bytes: u32, + }, + VarHeapMemory { + local_id: LocalId, + }, +} + +impl SymbolStorage { + pub fn local_id(&self) -> LocalId { + match self { + Self::ParamPrimitive { local_id, .. } => *local_id, + Self::ParamStackMemory { local_id, .. } => *local_id, + Self::VarPrimitive { local_id, .. } => *local_id, + Self::VarStackMemory { local_id, .. } => *local_id, + Self::VarHeapMemory { local_id, .. } => *local_id, + } + } + + pub fn value_type(&self) -> ValueType { + match self { + Self::ParamPrimitive { value_type, .. } => *value_type, + Self::VarPrimitive { value_type, .. } => *value_type, + Self::ParamStackMemory { .. } => ValueType::I32, + Self::VarStackMemory { .. } => ValueType::I32, + Self::VarHeapMemory { .. } => ValueType::I32, + } + } + + pub fn has_stack_memory(&self) -> bool { + match self { + Self::ParamStackMemory { .. } => true, + Self::VarStackMemory { .. } => true, + Self::ParamPrimitive { .. } => false, + Self::VarPrimitive { .. } => false, + Self::VarHeapMemory { .. } => false, + } + } + + pub fn stack_size_and_alignment(&self) -> (u32, u32) { + match self { + Self::VarStackMemory { + size, + alignment_bytes, + .. + } + | Self::ParamStackMemory { + size, + alignment_bytes, + .. + } => (*size, *alignment_bytes), + + _ => (0, 0), + } + } + + pub fn copy_to_memory( + &self, + instructions: &mut Vec, + to_pointer: LocalId, + to_offset: u32, + ) -> u32 { + match self { + Self::ParamPrimitive { + local_id, + value_type, + size, + .. + } + | Self::VarPrimitive { + local_id, + value_type, + size, + .. + } => { + let store_instruction = match (value_type, size) { + (ValueType::I64, 8) => I64Store(ALIGN_8, to_offset), + (ValueType::I32, 4) => I32Store(ALIGN_4, to_offset), + (ValueType::I32, 2) => I32Store16(ALIGN_2, to_offset), + (ValueType::I32, 1) => I32Store8(ALIGN_1, to_offset), + (ValueType::F32, 4) => F32Store(ALIGN_4, to_offset), + (ValueType::F64, 8) => F64Store(ALIGN_8, to_offset), + _ => { + panic!("Cannot store {:?} with alignment of {:?}", value_type, size); + } + }; + instructions.push(GetLocal(to_pointer.0)); + instructions.push(GetLocal(local_id.0)); + instructions.push(store_instruction); + *size + } + + Self::ParamStackMemory { + local_id, + size, + alignment_bytes, + } + | Self::VarStackMemory { + local_id, + size, + alignment_bytes, + .. + } => { + copy_memory( + instructions, + *local_id, + to_pointer, + *size, + *alignment_bytes, + to_offset, + ); + *size + } + + Self::VarHeapMemory { local_id, .. } => { + instructions.push(GetLocal(to_pointer.0)); + instructions.push(GetLocal(local_id.0)); + instructions.push(I32Store(ALIGN_4, to_offset)); + 4 + } + } + } +} diff --git a/compiler/gen_wasm/tests/helpers/eval.rs b/compiler/gen_wasm/tests/helpers/eval.rs index e809bbf075..f29a360094 100644 --- a/compiler/gen_wasm/tests/helpers/eval.rs +++ b/compiler/gen_wasm/tests/helpers/eval.rs @@ -1,6 +1,10 @@ use roc_can::builtins::builtin_defs_map; use roc_collections::all::{MutMap, MutSet}; // use roc_std::{RocDec, RocList, RocOrder, RocStr}; +use crate::helpers::wasm32_test_result::Wasm32TestResult; +use roc_gen_wasm::from_wasm32_memory::FromWasm32Memory; + +const TEST_WRAPPER_NAME: &str = "test_wrapper"; fn promote_expr_to_module(src: &str) -> String { let mut buffer = String::from("app \"test\" provides [ main ] to \"./platform\"\n\nmain =\n"); @@ -16,12 +20,11 @@ fn promote_expr_to_module(src: &str) -> String { } #[allow(dead_code)] -pub fn helper_wasm<'a>( +pub fn helper_wasm<'a, T: Wasm32TestResult>( arena: &'a bumpalo::Bump, src: &str, stdlib: &'a roc_builtins::std::StdLib, - _is_gen_test: bool, - _ignore_problems: bool, + _result_type_dummy: &T, ) -> wasmer::Instance { use std::path::{Path, PathBuf}; @@ -91,7 +94,11 @@ pub fn helper_wasm<'a>( exposed_to_host, }; - let module_bytes = roc_gen_wasm::build_module(&env, procedures).unwrap(); + let (mut builder, main_function_index) = + roc_gen_wasm::build_module_help(&env, procedures).unwrap(); + T::insert_test_wrapper(&mut builder, TEST_WRAPPER_NAME, main_function_index); + + let module_bytes = builder.build().to_bytes().unwrap(); // for debugging (e.g. with wasm2wat) if false { @@ -128,45 +135,40 @@ pub fn helper_wasm<'a>( } #[allow(dead_code)] -pub fn assert_wasm_evals_to_help(src: &str, ignore_problems: bool) -> Result +pub fn assert_wasm_evals_to_help(src: &str, expected: T) -> Result where - T: Copy, + T: FromWasm32Memory + Wasm32TestResult, { let arena = bumpalo::Bump::new(); // NOTE the stdlib must be in the arena; just taking a reference will segfault let stdlib = arena.alloc(roc_builtins::std::standard_stdlib()); - let is_gen_test = true; - let instance = - crate::helpers::eval::helper_wasm(&arena, src, stdlib, is_gen_test, ignore_problems); + let instance = crate::helpers::eval::helper_wasm(&arena, src, stdlib, &expected); - let main_function = instance.exports.get_function("#UserApp_main_1").unwrap(); + let memory = instance.exports.get_memory("memory").unwrap(); - match main_function.call(&[]) { + let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap(); + + match test_wrapper.call(&[]) { Err(e) => Err(format!("{:?}", e)), Ok(result) => { - let integer = match result[0] { - wasmer::Value::I32(a) => a as i64, - wasmer::Value::I64(a) => a, - wasmer::Value::F64(a) => a.to_bits() as i64, + let address = match result[0] { + wasmer::Value::I32(a) => a, _ => panic!(), }; - let output_ptr: &T; - unsafe { - output_ptr = std::mem::transmute::<&i64, &T>(&integer); - } + let output = ::decode(memory, address as u32); - Ok(*output_ptr) + Ok(output) } } } #[macro_export] macro_rules! assert_wasm_evals_to { - ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => { - match $crate::helpers::eval::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems) { + ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { + match $crate::helpers::eval::assert_wasm_evals_to_help::<$ty>($src, $expected) { Err(msg) => println!("{:?}", msg), Ok(actual) => { #[allow(clippy::bool_assert_comparison)] @@ -176,11 +178,11 @@ macro_rules! assert_wasm_evals_to { }; ($src:expr, $expected:expr, $ty:ty) => { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity); }; ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform); }; } @@ -192,7 +194,7 @@ macro_rules! assert_evals_to { ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { // Same as above, except with an additional transformation argument. { - $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false); + $crate::assert_wasm_evals_to!($src, $expected, $ty, $transform); } }; } diff --git a/compiler/gen_wasm/tests/helpers/mod.rs b/compiler/gen_wasm/tests/helpers/mod.rs index 7e538193f7..c28fee53d9 100644 --- a/compiler/gen_wasm/tests/helpers/mod.rs +++ b/compiler/gen_wasm/tests/helpers/mod.rs @@ -2,6 +2,7 @@ extern crate bumpalo; #[macro_use] pub mod eval; +pub mod wasm32_test_result; /// Used in the with_larger_debug_stack() function, for tests that otherwise /// run out of stack space in debug builds (but don't in --release builds) diff --git a/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs b/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs new file mode 100644 index 0000000000..5cf709a597 --- /dev/null +++ b/compiler/gen_wasm/tests/helpers/wasm32_test_result.rs @@ -0,0 +1,278 @@ +use parity_wasm::builder; +use parity_wasm::builder::ModuleBuilder; +use parity_wasm::elements::{ + Instruction, Instruction::*, Instructions, Internal, Local, ValueType, +}; + +use roc_gen_wasm::from_wasm32_memory::FromWasm32Memory; +use roc_gen_wasm::*; +use roc_std::{RocDec, RocList, RocOrder, RocStr}; + +const STACK_POINTER_LOCAL_ID: u32 = 0; + +pub trait Wasm32TestResult { + fn insert_test_wrapper( + module_builder: &mut ModuleBuilder, + wrapper_name: &str, + main_function_index: u32, + ) { + let instructions = Self::build_wrapper_body(main_function_index); + + let signature = builder::signature().with_result(ValueType::I32).build_sig(); + + let stack_frame_pointer = Local::new(1, ValueType::I32); + let function_def = builder::function() + .with_signature(signature) + .body() + .with_locals(vec![stack_frame_pointer]) + .with_instructions(Instructions::new(instructions)) + .build() // body + .build(); // function + + let location = module_builder.push_function(function_def); + let export = builder::export() + .field(wrapper_name) + .with_internal(Internal::Function(location.body)) + .build(); + + module_builder.push_export(export); + } + + fn build_wrapper_body(main_function_index: u32) -> Vec; +} + +macro_rules! build_wrapper_body_primitive { + ($store_instruction: expr, $align: expr) => { + fn build_wrapper_body(main_function_index: u32) -> Vec { + let size: i32 = 8; + let mut instructions = Vec::with_capacity(16); + push_stack_frame(&mut instructions, size, LocalId(STACK_POINTER_LOCAL_ID)); + instructions.extend([ + // load result address to prepare for the store instruction later + GetLocal(STACK_POINTER_LOCAL_ID), + // + // Call the main function with no arguments. Get primitive back. + Call(main_function_index), + // + // Store the primitive at the allocated address + $store_instruction($align, 0), + // + // Return the result pointer + GetLocal(STACK_POINTER_LOCAL_ID), + ]); + pop_stack_frame(&mut instructions, size, LocalId(STACK_POINTER_LOCAL_ID)); + instructions.push(End); + instructions + } + }; +} + +macro_rules! wasm_test_result_primitive { + ($type_name: ident, $store_instruction: expr, $align: expr) => { + impl Wasm32TestResult for $type_name { + build_wrapper_body_primitive!($store_instruction, $align); + } + }; +} + +fn build_wrapper_body_stack_memory(main_function_index: u32, size: usize) -> Vec { + let mut instructions = Vec::with_capacity(16); + push_stack_frame( + &mut instructions, + size as i32, + LocalId(STACK_POINTER_LOCAL_ID), + ); + instructions.extend([ + // + // Call the main function with the allocated address to write the result. + // No value is returned to the VM stack. This is the same as in compiled C. + GetLocal(STACK_POINTER_LOCAL_ID), + Call(main_function_index), + // + // Return the result address + GetLocal(STACK_POINTER_LOCAL_ID), + ]); + pop_stack_frame( + &mut instructions, + size as i32, + LocalId(STACK_POINTER_LOCAL_ID), + ); + instructions.push(End); + instructions +} + +macro_rules! wasm_test_result_stack_memory { + ($type_name: ident) => { + impl Wasm32TestResult for $type_name { + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, $type_name::ACTUAL_WIDTH) + } + } + }; +} + +wasm_test_result_primitive!(bool, I32Store8, ALIGN_1); +wasm_test_result_primitive!(RocOrder, I32Store8, ALIGN_1); + +wasm_test_result_primitive!(u8, I32Store8, ALIGN_1); +wasm_test_result_primitive!(i8, I32Store8, ALIGN_1); +wasm_test_result_primitive!(u16, I32Store16, ALIGN_2); +wasm_test_result_primitive!(i16, I32Store16, ALIGN_2); +wasm_test_result_primitive!(u32, I32Store, ALIGN_4); +wasm_test_result_primitive!(i32, I32Store, ALIGN_4); +wasm_test_result_primitive!(u64, I64Store, ALIGN_8); +wasm_test_result_primitive!(i64, I64Store, ALIGN_8); + +wasm_test_result_primitive!(f32, F32Store, ALIGN_8); +wasm_test_result_primitive!(f64, F64Store, ALIGN_8); + +wasm_test_result_stack_memory!(u128); +wasm_test_result_stack_memory!(i128); +wasm_test_result_stack_memory!(RocDec); +wasm_test_result_stack_memory!(RocStr); + +impl Wasm32TestResult for RocList { + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, 12) + } +} + +impl Wasm32TestResult for &'_ T { + build_wrapper_body_primitive!(I32Store, ALIGN_4); +} + +impl Wasm32TestResult for [T; N] +where + T: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, N * T::ACTUAL_WIDTH) + } +} + +impl Wasm32TestResult for (T, U) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory(main_function_index, T::ACTUAL_WIDTH + U::ACTUAL_WIDTH) + } +} + +impl Wasm32TestResult for (T, U, V) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + U::ACTUAL_WIDTH + V::ACTUAL_WIDTH, + ) + } +} + +impl Wasm32TestResult for (T, U, V, W) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, + W: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + U::ACTUAL_WIDTH + V::ACTUAL_WIDTH + W::ACTUAL_WIDTH, + ) + } +} + +impl Wasm32TestResult for (T, U, V, W, X) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, + W: Wasm32TestResult + FromWasm32Memory, + X: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + U::ACTUAL_WIDTH + V::ACTUAL_WIDTH + W::ACTUAL_WIDTH + X::ACTUAL_WIDTH, + ) + } +} + +impl Wasm32TestResult for (T, U, V, W, X, Y) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, + W: Wasm32TestResult + FromWasm32Memory, + X: Wasm32TestResult + FromWasm32Memory, + Y: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + + U::ACTUAL_WIDTH + + V::ACTUAL_WIDTH + + W::ACTUAL_WIDTH + + X::ACTUAL_WIDTH + + Y::ACTUAL_WIDTH, + ) + } +} + +impl Wasm32TestResult for (T, U, V, W, X, Y, Z) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, + W: Wasm32TestResult + FromWasm32Memory, + X: Wasm32TestResult + FromWasm32Memory, + Y: Wasm32TestResult + FromWasm32Memory, + Z: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + + U::ACTUAL_WIDTH + + V::ACTUAL_WIDTH + + W::ACTUAL_WIDTH + + X::ACTUAL_WIDTH + + Y::ACTUAL_WIDTH + + Z::ACTUAL_WIDTH, + ) + } +} + +impl Wasm32TestResult for (T, U, V, W, X, Y, Z, A) +where + T: Wasm32TestResult + FromWasm32Memory, + U: Wasm32TestResult + FromWasm32Memory, + V: Wasm32TestResult + FromWasm32Memory, + W: Wasm32TestResult + FromWasm32Memory, + X: Wasm32TestResult + FromWasm32Memory, + Y: Wasm32TestResult + FromWasm32Memory, + Z: Wasm32TestResult + FromWasm32Memory, + A: Wasm32TestResult + FromWasm32Memory, +{ + fn build_wrapper_body(main_function_index: u32) -> Vec { + build_wrapper_body_stack_memory( + main_function_index, + T::ACTUAL_WIDTH + + U::ACTUAL_WIDTH + + V::ACTUAL_WIDTH + + W::ACTUAL_WIDTH + + X::ACTUAL_WIDTH + + Y::ACTUAL_WIDTH + + Z::ACTUAL_WIDTH + + A::ACTUAL_WIDTH, + ) + } +} diff --git a/compiler/gen_wasm/tests/wasm_num.rs b/compiler/gen_wasm/tests/wasm_num.rs index a10d276c6c..bc327aa0eb 100644 --- a/compiler/gen_wasm/tests/wasm_num.rs +++ b/compiler/gen_wasm/tests/wasm_num.rs @@ -11,7 +11,7 @@ extern crate libc; mod helpers; #[cfg(all(test, any(target_os = "linux", target_os = "macos"), any(target_arch = "x86_64"/*, target_arch = "aarch64"*/)))] -mod dev_num { +mod wasm_num { #[test] fn i64_values() { assert_evals_to!("0", 0, i64); @@ -36,6 +36,101 @@ mod dev_num { assert_evals_to!(&format!("{:0.1}", f64::MAX), f64::MAX, f64); } + #[test] + fn i8_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I8 + x = 0x7f + 0x7f + + x + "# + ), + -2, + i8 + ); + } + + #[test] + fn i16_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I16 + x = 0x7fff + 0x7fff + + x + "# + ), + -2, + i16 + ); + } + + #[test] + fn i32_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : I32 + x = 0x7fffffff + 0x7fffffff + + x + "# + ), + -2, + i32 + ); + } + + #[test] + fn u8_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U8 + x = 0xff + 0xff + + x + "# + ), + 0xfe, + u8 + ); + } + + #[test] + fn u16_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U16 + x = 0xffff + 0xffff + + x + "# + ), + 0xfffe, + u16 + ); + } + #[test] + fn u32_add_wrap() { + assert_evals_to!( + indoc!( + r#" + x : U32 + x = 0xffffffff + 0xffffffff + + x + "# + ), + 0xfffffffe, + u32 + ); + } + #[test] fn gen_add_i64() { assert_evals_to!( @@ -154,44 +249,44 @@ mod dev_num { ); } - // #[test] - // fn gen_add_f64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1.1 + 2.4 + 3 - // "# - // ), - // 6.5, - // f64 - // ); - // } + #[test] + fn gen_add_f64() { + assert_evals_to!( + indoc!( + r#" + 1.1 + 2.4 + 3 + "# + ), + 6.5, + f64 + ); + } - // #[test] - // fn gen_sub_i64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 - 2 - 3 - // "# - // ), - // -4, - // i64 - // ); - // } + #[test] + fn gen_sub_i64() { + assert_evals_to!( + indoc!( + r#" + 1 - 2 - 3 + "# + ), + -4, + i64 + ); + } - // #[test] - // fn gen_mul_i64() { - // assert_evals_to!( - // indoc!( - // r#" - // 2 * 4 * 6 - // "# - // ), - // 48, - // i64 - // ); - // } + #[test] + fn gen_mul_i64() { + assert_evals_to!( + indoc!( + r#" + 2 * 4 * 6 + "# + ), + 48, + i64 + ); + } #[test] fn i64_force_stack() { @@ -476,18 +571,18 @@ mod dev_num { // ); // } - // #[test] - // fn gen_sub_f64() { - // assert_evals_to!( - // indoc!( - // r#" - // 1.5 - 2.4 - 3 - // "# - // ), - // -3.9, - // f64 - // ); - // } + #[test] + fn gen_sub_f64() { + assert_evals_to!( + indoc!( + r#" + 1.5 - 2.4 - 3 + "# + ), + -3.9, + f64 + ); + } // #[test] // fn gen_div_i64() { @@ -685,31 +780,31 @@ mod dev_num { // assert_evals_to!("0.0 >= 0.0", true, bool); // } - // #[test] - // fn gen_order_of_arithmetic_ops() { - // assert_evals_to!( - // indoc!( - // r#" - // 1 + 3 * 7 - 2 - // "# - // ), - // 20, - // i64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops() { + assert_evals_to!( + indoc!( + r#" + 1 + 3 * 7 - 2 + "# + ), + 20, + i64 + ); + } - // #[test] - // fn gen_order_of_arithmetic_ops_complex_float() { - // assert_evals_to!( - // indoc!( - // r#" - // 3 - 48 * 2.0 - // "# - // ), - // -93.0, - // f64 - // ); - // } + #[test] + fn gen_order_of_arithmetic_ops_complex_float() { + assert_evals_to!( + indoc!( + r#" + 3 - 48 * 2.0 + "# + ), + -93.0, + f64 + ); + } // #[test] // fn if_guard_bind_variable_false() { diff --git a/compiler/gen_wasm/tests/wasm_records.rs b/compiler/gen_wasm/tests/wasm_records.rs index 63f72357a2..884e92c7db 100644 --- a/compiler/gen_wasm/tests/wasm_records.rs +++ b/compiler/gen_wasm/tests/wasm_records.rs @@ -5,7 +5,7 @@ extern crate indoc; mod helpers; #[cfg(all(test, target_os = "linux", any(target_arch = "x86_64"/*, target_arch = "aarch64"*/)))] -mod dev_records { +mod wasm_records { // #[test] // fn basic_record() { // assert_evals_to!( @@ -307,142 +307,116 @@ mod dev_records { // () // ); // } - // - // #[test] - // fn i64_record1_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // { x: 3 } - // "# - // ), - // 3, - // i64 - // ); - // } - // #[test] - // fn i64_record2_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // { x: 3, y: 5 } - // "# - // ), - // (3, 5), - // (i64, i64) - // ); - // } + #[test] + fn i64_record1_literal() { + assert_evals_to!( + indoc!( + r#" + { x: 3 } + "# + ), + 3, + i64 + ); + } - // // #[test] - // // fn i64_record3_literal() { - // // assert_evals_to!( - // // indoc!( - // // r#" - // // { x: 3, y: 5, z: 17 } - // // "# - // // ), - // // (3, 5, 17), - // // (i64, i64, i64) - // // ); - // // } + #[test] + fn i64_record2_literal() { + assert_evals_to!( + indoc!( + r#" + { x: 3, y: 5 } + "# + ), + (3, 5), + (i64, i64) + ); + } - // #[test] - // fn f64_record2_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // { x: 3.1, y: 5.1 } - // "# - // ), - // (3.1, 5.1), - // (f64, f64) - // ); - // } + #[test] + fn i64_record3_literal() { + assert_evals_to!( + indoc!( + r#" + { x: 3, y: 5, z: 17 } + "# + ), + (3, 5, 17), + (i64, i64, i64) + ); + } - // // #[test] - // // fn f64_record3_literal() { - // // assert_evals_to!( - // // indoc!( - // // r#" - // // { x: 3.1, y: 5.1, z: 17.1 } - // // "# - // // ), - // // (3.1, 5.1, 17.1), - // // (f64, f64, f64) - // // ); - // // } + #[test] + fn f64_record2_literal() { + assert_evals_to!( + indoc!( + r#" + { x: 3.1, y: 5.1 } + "# + ), + (3.1, 5.1), + (f64, f64) + ); + } - // // #[test] - // // fn bool_record4_literal() { - // // assert_evals_to!( - // // indoc!( - // // r#" - // // record : { a : Bool, b : Bool, c : Bool, d : Bool } - // // record = { a: True, b: True, c : True, d : Bool } + #[test] + fn f64_record3_literal() { + assert_evals_to!( + indoc!( + r#" + { x: 3.1, y: 5.1, z: 17.1 } + "# + ), + (3.1, 5.1, 17.1), + (f64, f64, f64) + ); + } - // // record - // // "# - // // ), - // // (true, false, false, true), - // // (bool, bool, bool, bool) - // // ); - // // } + #[test] + fn bool_record4_literal() { + assert_evals_to!( + indoc!( + r#" + record : { a : Bool, b : Bool, c : Bool, d : Bool } + record = { a: True, b: False, c : False, d : True } - // #[test] - // fn i64_record1_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3 } - // "# - // ), - // 3, - // i64 - // ); - // } + record + "# + ), + [true, false, false, true], + [bool; 4] + ); + } - // // #[test] - // // fn i64_record9_literal() { - // // assert_evals_to!( - // // indoc!( - // // r#" - // // { a: 3, b: 5, c: 17, d: 1, e: 9, f: 12, g: 13, h: 14, i: 15 } - // // "# - // // ), - // // (3, 5, 17, 1, 9, 12, 13, 14, 15), - // // (i64, i64, i64, i64, i64, i64, i64, i64, i64) - // // ); - // // } + #[test] + fn i64_record9_literal() { + assert_evals_to!( + indoc!( + r#" + { a: 3, b: 5, c: 17, d: 1, e: 9, f: 12, g: 13, h: 14, i: 15 } + "# + ), + [3, 5, 17, 1, 9, 12, 13, 14, 15], + [i64; 9] + ); + } - // // #[test] - // // fn f64_record3_literal() { - // // assert_evals_to!( - // // indoc!( - // // r#" - // // { x: 3.1, y: 5.1, z: 17.1 } - // // "# - // // ), - // // (3.1, 5.1, 17.1), - // // (f64, f64, f64) - // // ); - // // } + #[test] + fn bool_literal() { + assert_evals_to!( + indoc!( + r#" + x : Bool + x = True - // #[test] - // fn bool_literal() { - // assert_evals_to!( - // indoc!( - // r#" - // x : Bool - // x = True - - // x - // "# - // ), - // true, - // bool - // ); - // } + x + "# + ), + true, + bool + ); + } // #[test] // fn optional_field_when_use_default() { @@ -667,135 +641,135 @@ mod dev_records { // ); // } - // #[test] - // fn return_record_2() { - // assert_evals_to!( - // indoc!( - // r#" - // { x: 3, y: 5 } - // "# - // ), - // [3, 5], - // [i64; 2] - // ); - // } + #[test] + fn return_record_2() { + assert_evals_to!( + indoc!( + r#" + { x: 3, y: 5 } + "# + ), + [3, 5], + [i64; 2] + ); + } - // #[test] - // fn return_record_3() { - // assert_evals_to!( - // indoc!( - // r#" - // { x: 3, y: 5, z: 4 } - // "# - // ), - // (3, 5, 4), - // (i64, i64, i64) - // ); - // } + #[test] + fn return_record_3() { + assert_evals_to!( + indoc!( + r#" + { x: 3, y: 5, z: 4 } + "# + ), + (3, 5, 4), + (i64, i64, i64) + ); + } - // #[test] - // fn return_record_4() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3, b: 5, c: 4, d: 2 } - // "# - // ), - // [3, 5, 4, 2], - // [i64; 4] - // ); - // } + #[test] + fn return_record_4() { + assert_evals_to!( + indoc!( + r#" + { a: 3, b: 5, c: 4, d: 2 } + "# + ), + [3, 5, 4, 2], + [i64; 4] + ); + } - // #[test] - // fn return_record_5() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3, b: 5, c: 4, d: 2, e: 1 } - // "# - // ), - // [3, 5, 4, 2, 1], - // [i64; 5] - // ); - // } + #[test] + fn return_record_5() { + assert_evals_to!( + indoc!( + r#" + { a: 3, b: 5, c: 4, d: 2, e: 1 } + "# + ), + [3, 5, 4, 2, 1], + [i64; 5] + ); + } - // #[test] - // fn return_record_6() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3, b: 5, c: 4, d: 2, e: 1, f: 7 } - // "# - // ), - // [3, 5, 4, 2, 1, 7], - // [i64; 6] - // ); - // } + #[test] + fn return_record_6() { + assert_evals_to!( + indoc!( + r#" + { a: 3, b: 5, c: 4, d: 2, e: 1, f: 7 } + "# + ), + [3, 5, 4, 2, 1, 7], + [i64; 6] + ); + } - // #[test] - // fn return_record_7() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3, b: 5, c: 4, d: 2, e: 1, f: 7, g: 8 } - // "# - // ), - // [3, 5, 4, 2, 1, 7, 8], - // [i64; 7] - // ); - // } + #[test] + fn return_record_7() { + assert_evals_to!( + indoc!( + r#" + { a: 3, b: 5, c: 4, d: 2, e: 1, f: 7, g: 8 } + "# + ), + [3, 5, 4, 2, 1, 7, 8], + [i64; 7] + ); + } - // #[test] - // fn return_record_float_int() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 3.14, b: 0x1 } - // "# - // ), - // (3.14, 0x1), - // (f64, i64) - // ); - // } + #[test] + fn return_record_float_int() { + assert_evals_to!( + indoc!( + r#" + { a: 3.14, b: 0x1 } + "# + ), + (3.14, 0x1), + (f64, i64) + ); + } - // #[test] - // fn return_record_int_float() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 0x1, b: 3.14 } - // "# - // ), - // (0x1, 3.14), - // (i64, f64) - // ); - // } + #[test] + fn return_record_int_float() { + assert_evals_to!( + indoc!( + r#" + { a: 0x1, b: 3.14 } + "# + ), + (0x1, 3.14), + (i64, f64) + ); + } - // #[test] - // fn return_record_float_float() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 6.28, b: 3.14 } - // "# - // ), - // (6.28, 3.14), - // (f64, f64) - // ); - // } + #[test] + fn return_record_float_float() { + assert_evals_to!( + indoc!( + r#" + { a: 6.28, b: 3.14 } + "# + ), + (6.28, 3.14), + (f64, f64) + ); + } - // #[test] - // fn return_record_float_float_float() { - // assert_evals_to!( - // indoc!( - // r#" - // { a: 6.28, b: 3.14, c: 0.1 } - // "# - // ), - // (6.28, 3.14, 0.1), - // (f64, f64, f64) - // ); - // } + #[test] + fn return_record_float_float_float() { + assert_evals_to!( + indoc!( + r#" + { a: 6.28, b: 3.14, c: 0.1 } + "# + ), + (6.28, 3.14, 0.1), + (f64, f64, f64) + ); + } // #[test] // fn return_nested_record() { @@ -851,20 +825,20 @@ mod dev_records { // ); // } - #[test] - fn update_single_element_record() { - assert_evals_to!( - indoc!( - r#" - rec = { foo: 42} + // #[test] + // fn update_single_element_record() { + // assert_evals_to!( + // indoc!( + // r#" + // rec = { foo: 42} - { rec & foo: rec.foo + 1 } - "# - ), - 43, - i64 - ); - } + // { rec & foo: rec.foo + 1 } + // "# + // ), + // 43, + // i64 + // ); + // } // #[test] // fn booleans_in_record() { @@ -899,6 +873,24 @@ mod dev_records { // ); // } + #[test] + fn stack_memory_return_from_branch() { + // stack memory pointer should end up in the right place after returning from a branch + assert_evals_to!( + indoc!( + r#" + stackMemoryJunk = { x: 999, y: 111 } + if True then + { x: 123, y: 321 } + else + stackMemoryJunk + "# + ), + (123, 321), + (i64, i64) + ); + } + // #[test] // fn blue_and_present() { // assert_evals_to!( diff --git a/compiler/load/src/file.rs b/compiler/load/src/file.rs index f848d2ce36..0a6fafe939 100644 --- a/compiler/load/src/file.rs +++ b/compiler/load/src/file.rs @@ -37,13 +37,13 @@ use roc_types::subs::{Subs, VarStore, Variable}; use roc_types::types::{Alias, Type}; use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::collections::{HashMap, HashSet}; -use std::fs; use std::io; use std::iter; use std::path::{Path, PathBuf}; use std::str::from_utf8_unchecked; use std::sync::Arc; use std::time::{Duration, SystemTime}; +use std::{env, fs}; /// Default name for the binary generated for an app, if an invalid one was specified. const DEFAULT_APP_OUTPUT_PATH: &str = "app"; @@ -1351,7 +1351,12 @@ where // doing .max(1) on the entire expression guards against // num_cpus returning 0, while also avoiding wrapping // unsigned subtraction overflow. - let num_workers = num_cpus::get().max(2) - 1; + let default_num_workers = num_cpus::get().max(2) - 1; + + let num_workers = match env::var("ROC_NUM_WORKERS") { + Ok(env_str) => env_str.parse::().unwrap_or(default_num_workers), + Err(_) => default_num_workers, + }; let worker_arenas = arena.alloc(bumpalo::collections::Vec::with_capacity_in( num_workers, @@ -2113,8 +2118,6 @@ fn update<'a>( &mut state.procedures, ); - Proc::insert_refcount_operations(arena, &mut state.procedures); - // display the mono IR of the module, for debug purposes if roc_mono::ir::PRETTY_PRINT_IR_SYMBOLS { let procs_string = state @@ -2128,6 +2131,8 @@ fn update<'a>( println!("{}", result); } + Proc::insert_refcount_operations(arena, &mut state.procedures); + // This is not safe with the new non-recursive RC updates that we do for tag unions // // Proc::optimize_refcount_operations( diff --git a/compiler/module/src/low_level.rs b/compiler/module/src/low_level.rs index 583e6017c1..7dec1f4ccf 100644 --- a/compiler/module/src/low_level.rs +++ b/compiler/module/src/low_level.rs @@ -15,6 +15,7 @@ pub enum LowLevel { StrFromUtf8, StrFromUtf8Range, StrToUtf8, + StrRepeat, StrFromFloat, ListLen, ListGetUnsafe, @@ -114,19 +115,19 @@ impl LowLevel { match self { StrConcat | StrJoinWith | StrIsEmpty | StrStartsWith | StrStartsWithCodePt | StrEndsWith | StrSplit | StrCountGraphemes | StrFromInt | StrFromUtf8 - | StrFromUtf8Range | StrToUtf8 | StrFromFloat | ListLen | ListGetUnsafe | ListSet - | ListDrop | ListSingle | ListRepeat | ListReverse | ListConcat | ListContains - | ListAppend | ListPrepend | ListJoin | ListRange | ListSwap | DictSize | DictEmpty - | DictInsert | DictRemove | DictContains | DictGetUnsafe | DictKeys | DictValues - | DictUnion | DictIntersection | DictDifference | SetFromList | NumAdd | NumAddWrap - | NumAddChecked | NumSub | NumSubWrap | NumSubChecked | NumMul | NumMulWrap - | NumMulChecked | NumGt | NumGte | NumLt | NumLte | NumCompare | NumDivUnchecked - | NumRemUnchecked | NumIsMultipleOf | NumAbs | NumNeg | NumSin | NumCos - | NumSqrtUnchecked | NumLogUnchecked | NumRound | NumToFloat | NumPow | NumCeiling - | NumPowInt | NumFloor | NumIsFinite | NumAtan | NumAcos | NumAsin | NumBitwiseAnd - | NumBitwiseXor | NumBitwiseOr | NumShiftLeftBy | NumShiftRightBy | NumBytesToU16 - | NumBytesToU32 | NumShiftRightZfBy | NumIntCast | Eq | NotEq | And | Or | Not - | Hash | ExpectTrue => false, + | StrFromUtf8Range | StrToUtf8 | StrRepeat | StrFromFloat | ListLen | ListGetUnsafe + | ListSet | ListDrop | ListSingle | ListRepeat | ListReverse | ListConcat + | ListContains | ListAppend | ListPrepend | ListJoin | ListRange | ListSwap + | DictSize | DictEmpty | DictInsert | DictRemove | DictContains | DictGetUnsafe + | DictKeys | DictValues | DictUnion | DictIntersection | DictDifference + | SetFromList | NumAdd | NumAddWrap | NumAddChecked | NumSub | NumSubWrap + | NumSubChecked | NumMul | NumMulWrap | NumMulChecked | NumGt | NumGte | NumLt + | NumLte | NumCompare | NumDivUnchecked | NumRemUnchecked | NumIsMultipleOf + | NumAbs | NumNeg | NumSin | NumCos | NumSqrtUnchecked | NumLogUnchecked | NumRound + | NumToFloat | NumPow | NumCeiling | NumPowInt | NumFloor | NumIsFinite | NumAtan + | NumAcos | NumAsin | NumBitwiseAnd | NumBitwiseXor | NumBitwiseOr | NumShiftLeftBy + | NumShiftRightBy | NumBytesToU16 | NumBytesToU32 | NumShiftRightZfBy | NumIntCast + | Eq | NotEq | And | Or | Not | Hash | ExpectTrue => false, ListMap | ListMap2 | ListMap3 | ListMapWithIndex | ListKeepIf | ListWalk | ListWalkUntil | ListWalkBackwards | ListKeepOks | ListKeepErrs | ListSortWith diff --git a/compiler/module/src/symbol.rs b/compiler/module/src/symbol.rs index aaa9ce321d..6e825a9953 100644 --- a/compiler/module/src/symbol.rs +++ b/compiler/module/src/symbol.rs @@ -927,6 +927,7 @@ define_builtins! { 16 STR_STARTS_WITH_CODE_PT: "startsWithCodePt" 17 STR_ALIAS_ANALYSIS_STATIC: "#aliasAnalysisStatic" // string with the static lifetime 18 STR_FROM_UTF8_RANGE: "fromUtf8Range" + 19 STR_REPEAT: "repeat" } 4 LIST: "List" => { 0 LIST_LIST: "List" imported // the List.List type alias diff --git a/compiler/mono/src/alias_analysis.rs b/compiler/mono/src/alias_analysis.rs index 229ed7cbee..bff15284a7 100644 --- a/compiler/mono/src/alias_analysis.rs +++ b/compiler/mono/src/alias_analysis.rs @@ -16,6 +16,7 @@ use crate::layout::{Builtin, Layout, ListLayout, UnionLayout}; pub const MOD_APP: ModName = ModName(b"UserApp"); pub const STATIC_STR_NAME: ConstName = ConstName(&Symbol::STR_ALIAS_ANALYSIS_STATIC.to_ne_bytes()); +pub const STATIC_LIST_NAME: ConstName = ConstName(b"THIS IS A STATIC LIST"); const ENTRY_POINT_NAME: &[u8] = b"mainForHost"; @@ -128,6 +129,22 @@ where }; m.add_const(STATIC_STR_NAME, static_str_def)?; + // a const that models all static lists + let static_list_def = { + let mut cbuilder = ConstDefBuilder::new(); + let block = cbuilder.add_block(); + let cell = cbuilder.add_new_heap_cell(block)?; + + let unit_type = cbuilder.add_tuple_type(&[])?; + let bag = cbuilder.add_empty_bag(block, unit_type)?; + let value_id = cbuilder.add_make_tuple(block, &[cell, bag])?; + let root = BlockExpr(block, value_id); + let list_type_id = static_list_type(&mut cbuilder)?; + + cbuilder.build(list_type_id, root)? + }; + m.add_const(STATIC_LIST_NAME, static_list_def)?; + // the entry point wrapper let roc_main_bytes = func_name_bytes_help( entry_point.symbol, @@ -589,9 +606,9 @@ fn call_spec( let index = builder.add_make_tuple(block, &[])?; let argument = if closure_env_layout.is_none() { - builder.add_make_tuple(block, &[first, index])? + builder.add_make_tuple(block, &[index, first])? } else { - builder.add_make_tuple(block, &[first, index, closure_env])? + builder.add_make_tuple(block, &[index, first, closure_env])? }; builder.add_call(block, spec_var, module, name, argument)?; } @@ -1117,9 +1134,11 @@ fn expr_spec<'a>( let list = new_list(builder, block, type_id)?; let mut bag = builder.add_get_tuple_field(block, list, LIST_BAG_INDEX)?; + let mut all_constants = true; for element in elems.iter() { let value_id = if let ListLiteralElement::Symbol(symbol) = element { + all_constants = false; env.symbols[symbol] } else { builder.add_make_tuple(block, &[]).unwrap() @@ -1128,9 +1147,13 @@ fn expr_spec<'a>( bag = builder.add_bag_insert(block, bag, value_id)?; } - let cell = builder.add_new_heap_cell(block)?; + if all_constants { + new_static_list(builder, block) + } else { + let cell = builder.add_new_heap_cell(block)?; - builder.add_make_tuple(block, &[cell, bag]) + builder.add_make_tuple(block, &[cell, bag]) + } } EmptyArray => { @@ -1191,6 +1214,11 @@ fn layout_spec_help( match layout { Builtin(builtin) => builtin_spec(builder, builtin, when_recursive), Struct(fields) => build_recursive_tuple_type(builder, fields, when_recursive), + LambdaSet(lambda_set) => layout_spec_help( + builder, + &lambda_set.runtime_representation(), + when_recursive, + ), Union(union_layout) => { let variant_types = build_variant_types(builder, union_layout)?; @@ -1236,7 +1264,7 @@ fn builtin_spec( match builtin { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize => builder.add_tuple_type(&[]), - Decimal | Float128 | Float64 | Float32 | Float16 => builder.add_tuple_type(&[]), + Decimal | Float128 | Float64 | Float32 => builder.add_tuple_type(&[]), Str | EmptyStr => str_type(builder), Dict(key_layout, value_layout) => { let value_type = layout_spec_help(builder, value_layout, when_recursive)?; @@ -1291,6 +1319,14 @@ fn str_type(builder: &mut TC) -> Result { builder.add_tuple_type(&[cell_id]) } +fn static_list_type(builder: &mut TC) -> Result { + let unit_type = builder.add_tuple_type(&[])?; + let cell = builder.add_heap_cell_type(); + let bag = builder.add_bag_type(unit_type)?; + + builder.add_tuple_type(&[cell, bag]) +} + // const OK_TAG_ID: u8 = 1u8; // const ERR_TAG_ID: u8 = 0u8; @@ -1324,6 +1360,12 @@ fn new_static_string(builder: &mut FuncDefBuilder, block: BlockId) -> Result Result { + let module = MOD_APP; + + builder.add_const_ref(block, module, STATIC_LIST_NAME) +} + fn new_num(builder: &mut FuncDefBuilder, block: BlockId) -> Result { // we model all our numbers as unit values builder.add_make_tuple(block, &[]) diff --git a/compiler/mono/src/borrow.rs b/compiler/mono/src/borrow.rs index 89d6ae2446..45f5eb9c0d 100644 --- a/compiler/mono/src/borrow.rs +++ b/compiler/mono/src/borrow.rs @@ -1013,6 +1013,7 @@ pub fn lowlevel_borrow_signature(arena: &Bump, op: LowLevel) -> &[bool] { StrFromUtf8 => arena.alloc_slice_copy(&[owned]), StrFromUtf8Range => arena.alloc_slice_copy(&[borrowed, irrelevant]), StrToUtf8 => arena.alloc_slice_copy(&[owned]), + StrRepeat => arena.alloc_slice_copy(&[borrowed, irrelevant]), StrFromInt | StrFromFloat => arena.alloc_slice_copy(&[irrelevant]), Hash => arena.alloc_slice_copy(&[borrowed, irrelevant]), DictSize => arena.alloc_slice_copy(&[borrowed]), diff --git a/compiler/mono/src/decision_tree.rs b/compiler/mono/src/decision_tree.rs index 436eb43446..16b3ea0a84 100644 --- a/compiler/mono/src/decision_tree.rs +++ b/compiler/mono/src/decision_tree.rs @@ -1,6 +1,7 @@ use crate::exhaustive::{Ctor, RenderAs, TagId, Union}; use crate::ir::{ - BranchInfo, DestructType, Env, Expr, JoinPointId, Literal, Param, Pattern, Procs, Stmt, + BranchInfo, DestructType, Env, Expr, FloatPrecision, IntPrecision, JoinPointId, Literal, Param, + Pattern, Procs, Stmt, }; use crate::layout::{Builtin, Layout, LayoutCache, UnionLayout}; use roc_collections::all::{MutMap, MutSet}; @@ -85,8 +86,8 @@ enum Test<'a> { union: crate::exhaustive::Union, arguments: Vec<(Pattern<'a>, Layout<'a>)>, }, - IsInt(i128), - IsFloat(u64), + IsInt(i128, IntPrecision), + IsFloat(u64, FloatPrecision), IsDecimal(RocDec), IsStr(Box), IsBit(bool), @@ -95,6 +96,7 @@ enum Test<'a> { num_alts: usize, }, } + use std::hash::{Hash, Hasher}; impl<'a> Hash for Test<'a> { fn hash(&self, state: &mut H) { @@ -106,13 +108,15 @@ impl<'a> Hash for Test<'a> { tag_id.hash(state); // The point of this custom implementation is to not hash the tag arguments } - IsInt(v) => { + IsInt(v, width) => { state.write_u8(1); v.hash(state); + width.hash(state); } - IsFloat(v) => { + IsFloat(v, width) => { state.write_u8(2); v.hash(state); + width.hash(state); } IsStr(v) => { state.write_u8(3); @@ -306,8 +310,8 @@ fn tests_are_complete_help(last_test: &Test, number_of_tests: usize) -> bool { Test::IsCtor { union, .. } => number_of_tests == union.alternatives.len(), Test::IsByte { num_alts, .. } => number_of_tests == *num_alts, Test::IsBit(_) => number_of_tests == 2, - Test::IsInt(_) => false, - Test::IsFloat(_) => false, + Test::IsInt(_, _) => false, + Test::IsFloat(_, _) => false, Test::IsDecimal(_) => false, Test::IsStr(_) => false, } @@ -561,8 +565,8 @@ fn test_at_path<'a>( tag_id: *tag_id, num_alts: union.alternatives.len(), }, - IntLiteral(v) => IsInt(*v), - FloatLiteral(v) => IsFloat(*v), + IntLiteral(v, precision) => IsInt(*v, *precision), + FloatLiteral(v, precision) => IsFloat(*v, *precision), DecimalLiteral(v) => IsDecimal(*v), StrLiteral(v) => IsStr(v.clone()), }; @@ -807,8 +811,9 @@ fn to_relevant_branch_help<'a>( _ => None, }, - IntLiteral(int) => match test { - IsInt(is_int) if int == *is_int => { + IntLiteral(int, p1) => match test { + IsInt(is_int, p2) if int == *is_int => { + debug_assert_eq!(p1, *p2); start.extend(end); Some(Branch { goal: branch.goal, @@ -819,8 +824,9 @@ fn to_relevant_branch_help<'a>( _ => None, }, - FloatLiteral(float) => match test { - IsFloat(test_float) if float == *test_float => { + FloatLiteral(float, p1) => match test { + IsFloat(test_float, p2) if float == *test_float => { + debug_assert_eq!(p1, *p2); start.extend(end); Some(Branch { goal: branch.goal, @@ -928,8 +934,8 @@ fn needs_tests(pattern: &Pattern) -> bool { | AppliedTag { .. } | BitLiteral { .. } | EnumLiteral { .. } - | IntLiteral(_) - | FloatLiteral(_) + | IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | StrLiteral(_) => true, } @@ -1280,22 +1286,22 @@ fn test_to_equality<'a>( _ => unreachable!("{:?}", (cond_layout, union)), } } - Test::IsInt(test_int) => { + Test::IsInt(test_int, precision) => { // TODO don't downcast i128 here debug_assert!(test_int <= i64::MAX as i128); let lhs = Expr::Literal(Literal::Int(test_int as i128)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Int64), lhs)); + stores.push((lhs_symbol, precision.as_layout(), lhs)); (stores, lhs_symbol, rhs_symbol, None) } - Test::IsFloat(test_int) => { + Test::IsFloat(test_int, precision) => { // TODO maybe we can actually use i64 comparison here? let test_float = f64::from_bits(test_int as u64); let lhs = Expr::Literal(Literal::Float(test_float)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Float64), lhs)); + stores.push((lhs_symbol, precision.as_layout(), lhs)); (stores, lhs_symbol, rhs_symbol, None) } @@ -1303,7 +1309,7 @@ fn test_to_equality<'a>( Test::IsDecimal(test_dec) => { let lhs = Expr::Literal(Literal::Int(test_dec.0)); let lhs_symbol = env.unique_symbol(); - stores.push((lhs_symbol, Layout::Builtin(Builtin::Int128), lhs)); + stores.push((lhs_symbol, *cond_layout, lhs)); (stores, lhs_symbol, rhs_symbol, None) } @@ -1737,8 +1743,8 @@ fn decide_to_branching<'a>( ); let tag = match test { - Test::IsInt(v) => v as u64, - Test::IsFloat(v) => v as u64, + Test::IsInt(v, _) => v as u64, + Test::IsFloat(v, _) => v as u64, Test::IsBit(v) => v as u64, Test::IsByte { tag_id, .. } => tag_id as u64, Test::IsCtor { tag_id, .. } => tag_id as u64, diff --git a/compiler/mono/src/exhaustive.rs b/compiler/mono/src/exhaustive.rs index 51b20d32aa..77c28d64a1 100644 --- a/compiler/mono/src/exhaustive.rs +++ b/compiler/mono/src/exhaustive.rs @@ -65,8 +65,8 @@ fn simplify(pattern: &crate::ir::Pattern) -> Pattern { use crate::ir::Pattern::*; match pattern { - IntLiteral(v) => Literal(Literal::Int(*v)), - FloatLiteral(v) => Literal(Literal::Float(*v)), + IntLiteral(v, _) => Literal(Literal::Int(*v)), + FloatLiteral(v, _) => Literal(Literal::Float(*v)), DecimalLiteral(v) => Literal(Literal::Decimal(*v)), StrLiteral(v) => Literal(Literal::Str(v.clone())), diff --git a/compiler/mono/src/ir.rs b/compiler/mono/src/ir.rs index 93eab5ffc2..bc620bdf74 100644 --- a/compiler/mono/src/ir.rs +++ b/compiler/mono/src/ir.rs @@ -54,6 +54,7 @@ macro_rules! return_on_layout_error_help { #[derive(Debug, Clone, Copy)] pub enum OptLevel { + Development, Normal, Optimize, } @@ -694,7 +695,20 @@ impl<'a> Procs<'a> { // the `layout` is a function pointer, while `_ignore_layout` can be a // closure. We only specialize functions, storing this value with a closure // layout will give trouble. - self.specialized.insert((symbol, layout), Done(proc)); + let arguments = + Vec::from_iter_in(proc.args.iter().map(|(l, _)| *l), env.arena) + .into_bump_slice(); + + let proper_layout = ProcLayout { + arguments, + result: proc.ret_layout, + }; + + // NOTE: some function are specialized to have a closure, but don't actually + // need any closure argument. Here is where we correct this sort of thing, + // by trusting the layout of the Proc, not of what we specialize for + self.specialized.remove(&(symbol, layout)); + self.specialized.insert((symbol, proper_layout), Done(proc)); } Err(error) => { panic!("TODO generate a RuntimeError message for {:?}", error); @@ -1843,7 +1857,7 @@ fn generate_runtime_error_function<'a>( args.push((*arg, env.unique_symbol())); } - args.push((lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)); + args.push((Layout::LambdaSet(lambda_set), Symbol::ARG_CLOSURE)); (args.into_bump_slice(), *ret_layout) } @@ -1919,7 +1933,29 @@ fn specialize_external<'a>( match layout { RawFunctionLayout::Function(argument_layouts, lambda_set, return_layout) => { let assigned = env.unique_symbol(); - let unit = env.unique_symbol(); + + let mut argument_symbols = + Vec::with_capacity_in(argument_layouts.len(), env.arena); + let mut proc_arguments = + Vec::with_capacity_in(argument_layouts.len() + 1, env.arena); + let mut top_level_arguments = + Vec::with_capacity_in(argument_layouts.len() + 1, env.arena); + + for layout in argument_layouts { + let symbol = env.unique_symbol(); + + proc_arguments.push((*layout, symbol)); + + argument_symbols.push(symbol); + top_level_arguments.push(*layout); + } + + // the proc needs to take an extra closure argument + let lambda_set_layout = Layout::LambdaSet(lambda_set); + proc_arguments.push((lambda_set_layout, Symbol::ARG_CLOSURE)); + + // this should also be reflected in the TopLevel signature + top_level_arguments.push(lambda_set_layout); let hole = env.arena.alloc(Stmt::Ret(assigned)); @@ -1927,20 +1963,16 @@ fn specialize_external<'a>( env, lambda_set, Symbol::ARG_CLOSURE, - env.arena.alloc([unit]), + argument_symbols.into_bump_slice(), argument_layouts, *return_layout, assigned, hole, ); - let body = let_empty_struct(unit, env.arena.alloc(body)); - let proc = Proc { name, - args: env - .arena - .alloc([(lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)]), + args: proc_arguments.into_bump_slice(), body, closure_data_layout: None, ret_layout: *return_layout, @@ -1951,7 +1983,7 @@ fn specialize_external<'a>( let top_level = ProcLayout::new( env.arena, - env.arena.alloc([lambda_set.runtime_representation()]), + top_level_arguments.into_bump_slice(), *return_layout, ); @@ -1999,7 +2031,7 @@ fn specialize_external<'a>( env.subs.rollback_to(snapshot); let closure_data_layout = match opt_closure_layout { - Some(closure_layout) => closure_layout.runtime_representation(), + Some(lambda_set) => Layout::LambdaSet(lambda_set), None => Layout::Struct(&[]), }; @@ -2149,7 +2181,7 @@ fn specialize_external<'a>( env.subs.rollback_to(snapshot); let closure_data_layout = match opt_closure_layout { - Some(closure_layout) => Some(closure_layout.runtime_representation()), + Some(lambda_set) => Some(Layout::LambdaSet(lambda_set)), None => None, }; @@ -2260,7 +2292,7 @@ fn build_specialized_proc<'a>( Some(lambda_set) if pattern_symbols.last() == Some(&Symbol::ARG_CLOSURE) => { // here we define the lifted (now top-level) f function. Its final argument is `Symbol::ARG_CLOSURE`, // it stores the closure structure (just an integer in this case) - proc_args.push((lambda_set.runtime_representation(), Symbol::ARG_CLOSURE)); + proc_args.push((Layout::LambdaSet(lambda_set), Symbol::ARG_CLOSURE)); debug_assert_eq!( pattern_layouts_len + 1, @@ -2297,7 +2329,7 @@ fn build_specialized_proc<'a>( } Ordering::Greater => { if pattern_symbols.is_empty() { - let ret_layout = lambda_set.runtime_representation(); + let ret_layout = Layout::LambdaSet(lambda_set); Ok(FunctionPointerBody { closure: None, ret_layout, @@ -2482,7 +2514,7 @@ where let raw = if procs.module_thunks.contains(&proc_name) { match raw { RawFunctionLayout::Function(_, lambda_set, _) => { - RawFunctionLayout::ZeroArgumentThunk(lambda_set.runtime_representation()) + RawFunctionLayout::ZeroArgumentThunk(Layout::LambdaSet(lambda_set)) } _ => raw, } @@ -2656,6 +2688,7 @@ macro_rules! match_on_closure_argument { let ret_layout = top_level.result; + match closure_data_layout { RawFunctionLayout::Function(_, lambda_set, _) => { lowlevel_match_on_lambda_set( @@ -2760,13 +2793,13 @@ pub fn with_hole<'a>( IntOrFloat::SignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(int)), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::UnsignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(int)), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), _ => unreachable!("unexpected float precision for integer"), @@ -2778,7 +2811,7 @@ pub fn with_hole<'a>( IntOrFloat::BinaryFloatType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Float(float)), - Layout::Builtin(float_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::DecimalFloatType => { @@ -2810,19 +2843,19 @@ pub fn with_hole<'a>( IntOrFloat::SignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(num.into())), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::UnsignedIntType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Int(num.into())), - Layout::Builtin(int_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::BinaryFloatType(precision) => Stmt::Let( assigned, Expr::Literal(Literal::Float(num as f64)), - Layout::Builtin(float_precision_to_builtin(precision)), + precision.as_layout(), hole, ), IntOrFloat::DecimalFloatType => { @@ -4131,6 +4164,8 @@ fn construct_closure_data<'a>( assigned: Symbol, hole: &'a Stmt<'a>, ) -> Stmt<'a> { + let lambda_set_layout = Layout::LambdaSet(lambda_set); + match lambda_set.layout_for_member(name) { ClosureRepresentation::Union { tag_id, @@ -4162,12 +4197,7 @@ fn construct_closure_data<'a>( arguments: symbols, }; - Stmt::Let( - assigned, - expr, - lambda_set.runtime_representation(), - env.arena.alloc(hole), - ) + Stmt::Let(assigned, expr, lambda_set_layout, env.arena.alloc(hole)) } ClosureRepresentation::AlphabeticOrderStruct(field_layouts) => { debug_assert_eq!(field_layouts.len(), symbols.len()); @@ -4198,7 +4228,7 @@ fn construct_closure_data<'a>( let expr = Expr::Struct(symbols); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } ClosureRepresentation::Other(Layout::Builtin(Builtin::Int1)) => { debug_assert_eq!(symbols.len(), 0); @@ -4207,7 +4237,7 @@ fn construct_closure_data<'a>( let tag_id = name != lambda_set.set[0].0; let expr = Expr::Literal(Literal::Bool(tag_id)); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } ClosureRepresentation::Other(Layout::Builtin(Builtin::Int8)) => { debug_assert_eq!(symbols.len(), 0); @@ -4216,7 +4246,7 @@ fn construct_closure_data<'a>( let tag_id = lambda_set.set.iter().position(|(s, _)| *s == name).unwrap() as u8; let expr = Expr::Literal(Literal::Byte(tag_id)); - Stmt::Let(assigned, expr, lambda_set.runtime_representation(), hole) + Stmt::Let(assigned, expr, lambda_set_layout, hole) } _ => unreachable!(), } @@ -5623,8 +5653,8 @@ fn store_pattern_help<'a>( // do nothing return StorePattern::NotProductive(stmt); } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5758,8 +5788,8 @@ fn store_tag_pattern<'a>( Underscore => { // ignore } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5834,8 +5864,8 @@ fn store_newtype_pattern<'a>( Underscore => { // ignore } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -5910,8 +5940,8 @@ fn store_record_destruct<'a>( // internally. But `y` is never used, so we must make sure it't not stored/loaded. return StorePattern::NotProductive(stmt); } - IntLiteral(_) - | FloatLiteral(_) + IntLiteral(_, _) + | FloatLiteral(_, _) | DecimalLiteral(_) | EnumLiteral { .. } | BitLiteral { .. } @@ -6060,7 +6090,7 @@ fn reuse_function_symbol<'a>( let layout = match raw { RawFunctionLayout::ZeroArgumentThunk(layout) => layout, RawFunctionLayout::Function(_, lambda_set, _) => { - lambda_set.runtime_representation() + Layout::LambdaSet(lambda_set) } }; @@ -6158,7 +6188,7 @@ fn reuse_function_symbol<'a>( // TODO suspicious // let layout = Layout::Closure(argument_layouts, lambda_set, ret_layout); // panic!("suspicious"); - let layout = lambda_set.runtime_representation(); + let layout = Layout::LambdaSet(lambda_set); let top_level = ProcLayout::new(env.arena, &[], layout); procs.insert_passed_by_name( env, @@ -6178,9 +6208,14 @@ fn reuse_function_symbol<'a>( layout_cache, ); - // a function name (non-closure) that is passed along - // it never has closure data, so we use the empty struct - return let_empty_struct(symbol, env.arena.alloc(result)); + construct_closure_data( + env, + lambda_set, + original, + &[], + symbol, + env.arena.alloc(result), + ) } } RawFunctionLayout::ZeroArgumentThunk(ret_layout) => { @@ -6348,7 +6383,7 @@ fn call_by_name<'a>( procs, fn_var, proc_name, - env.arena.alloc(lambda_set.runtime_representation()), + env.arena.alloc(Layout::LambdaSet(lambda_set)), layout_cache, assigned, hole, @@ -6392,7 +6427,7 @@ fn call_by_name<'a>( procs, fn_var, proc_name, - env.arena.alloc(lambda_set.runtime_representation()), + env.arena.alloc(Layout::LambdaSet(lambda_set)), layout_cache, closure_data_symbol, env.arena.alloc(result), @@ -6522,7 +6557,7 @@ fn call_by_name_help<'a>( force_thunk( env, proc_name, - lambda_set.runtime_representation(), + Layout::LambdaSet(lambda_set), assigned, hole, ) @@ -6836,13 +6871,7 @@ fn call_specialized_proc<'a>( arguments: field_symbols, }; - build_call( - env, - call, - assigned, - lambda_set.runtime_representation(), - hole, - ) + build_call(env, call, assigned, Layout::LambdaSet(lambda_set), hole) } RawFunctionLayout::ZeroArgumentThunk(_) => { unreachable!() @@ -6882,8 +6911,8 @@ fn call_specialized_proc<'a>( pub enum Pattern<'a> { Identifier(Symbol), Underscore, - IntLiteral(i128), - FloatLiteral(u64), + IntLiteral(i128, IntPrecision), + FloatLiteral(u64, FloatPrecision), DecimalLiteral(RocDec), BitLiteral { value: bool, @@ -6961,22 +6990,36 @@ fn from_can_pattern_help<'a>( match can_pattern { Underscore => Ok(Pattern::Underscore), Identifier(symbol) => Ok(Pattern::Identifier(*symbol)), - IntLiteral(_, _, int) => Ok(Pattern::IntLiteral(*int as i128)), + IntLiteral(var, _, int) => { + match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, false) { + IntOrFloat::SignedIntType(precision) | IntOrFloat::UnsignedIntType(precision) => { + Ok(Pattern::IntLiteral(*int as i128, precision)) + } + other => { + panic!( + "Invalid precision for int pattern: {:?} has {:?}", + can_pattern, other + ) + } + } + } FloatLiteral(var, float_str, float) => { // TODO: Can I reuse num_argument_to_int_or_float here if I pass in true? match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, true) { - IntOrFloat::SignedIntType(_) => { - panic!("Invalid percision for float literal = {:?}", var) + IntOrFloat::SignedIntType(_) | IntOrFloat::UnsignedIntType(_) => { + panic!("Invalid precision for float pattern {:?}", var) } - IntOrFloat::UnsignedIntType(_) => { - panic!("Invalid percision for float literal = {:?}", var) + IntOrFloat::BinaryFloatType(precision) => { + Ok(Pattern::FloatLiteral(f64::to_bits(*float), precision)) } - IntOrFloat::BinaryFloatType(_) => Ok(Pattern::FloatLiteral(f64::to_bits(*float))), IntOrFloat::DecimalFloatType => { let dec = match RocDec::from_str(float_str) { - Some(d) => d, - None => panic!("Invalid decimal for float literal = {}. TODO: Make this a nice, user-friendly error message", float_str), - }; + Some(d) => d, + None => panic!( + r"Invalid decimal for float literal = {}. TODO: Make this a nice, user-friendly error message", + float_str + ), + }; Ok(Pattern::DecimalLiteral(dec)) } } @@ -6993,9 +7036,15 @@ fn from_can_pattern_help<'a>( } NumLiteral(var, num_str, num) => { match num_argument_to_int_or_float(env.subs, env.ptr_bytes, *var, false) { - IntOrFloat::SignedIntType(_) => Ok(Pattern::IntLiteral(*num as i128)), - IntOrFloat::UnsignedIntType(_) => Ok(Pattern::IntLiteral(*num as i128)), - IntOrFloat::BinaryFloatType(_) => Ok(Pattern::FloatLiteral(*num as u64)), + IntOrFloat::SignedIntType(precision) => { + Ok(Pattern::IntLiteral(*num as i128, precision)) + } + IntOrFloat::UnsignedIntType(precision) => { + Ok(Pattern::IntLiteral(*num as i128, precision)) + } + IntOrFloat::BinaryFloatType(precision) => { + Ok(Pattern::FloatLiteral(*num as u64, precision)) + } IntOrFloat::DecimalFloatType => { let dec = match RocDec::from_str(num_str) { Some(d) => d, @@ -7577,7 +7626,7 @@ fn from_can_record_destruct<'a>( }) } -#[derive(Debug)] +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum IntPrecision { Usize, I128, @@ -7587,11 +7636,45 @@ pub enum IntPrecision { I8, } +impl IntPrecision { + pub fn as_layout(&self) -> Layout<'static> { + Layout::Builtin(self.as_builtin()) + } + + pub fn as_builtin(&self) -> Builtin<'static> { + use IntPrecision::*; + match self { + I128 => Builtin::Int128, + I64 => Builtin::Int64, + I32 => Builtin::Int32, + I16 => Builtin::Int16, + I8 => Builtin::Int8, + Usize => Builtin::Usize, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Hash)] pub enum FloatPrecision { F64, F32, } +impl FloatPrecision { + pub fn as_layout(&self) -> Layout<'static> { + Layout::Builtin(self.as_builtin()) + } + + pub fn as_builtin(&self) -> Builtin<'static> { + use FloatPrecision::*; + match self { + F64 => Builtin::Float64, + F32 => Builtin::Float32, + } + } +} + +#[derive(Debug)] pub enum IntOrFloat { SignedIntType(IntPrecision), UnsignedIntType(IntPrecision), @@ -7599,26 +7682,6 @@ pub enum IntOrFloat { DecimalFloatType, } -fn float_precision_to_builtin(precision: FloatPrecision) -> Builtin<'static> { - use FloatPrecision::*; - match precision { - F64 => Builtin::Float64, - F32 => Builtin::Float32, - } -} - -fn int_precision_to_builtin(precision: IntPrecision) -> Builtin<'static> { - use IntPrecision::*; - match precision { - I128 => Builtin::Int128, - I64 => Builtin::Int64, - I32 => Builtin::Int32, - I16 => Builtin::Int16, - I8 => Builtin::Int8, - Usize => Builtin::Usize, - } -} - /// Given the `a` in `Num a`, determines whether it's an int or a float pub fn num_argument_to_int_or_float( subs: &Subs, @@ -7915,8 +7978,8 @@ fn match_on_lambda_set<'a>( let result = union_lambda_set_to_switch( env, - lambda_set.set, - lambda_set.runtime_representation(), + lambda_set, + Layout::Union(union_layout), closure_tag_id_symbol, union_layout.tag_id_layout(), closure_data_symbol, @@ -7946,6 +8009,7 @@ fn match_on_lambda_set<'a>( union_lambda_set_branch_help( env, function_symbol, + lambda_set, closure_data_symbol, Layout::Struct(fields), argument_symbols, @@ -7994,7 +8058,7 @@ fn match_on_lambda_set<'a>( #[allow(clippy::too_many_arguments)] fn union_lambda_set_to_switch<'a>( env: &mut Env<'a, '_>, - lambda_set: &'a [(Symbol, &'a [Layout<'a>])], + lambda_set: LambdaSet<'a>, closure_layout: Layout<'a>, closure_tag_id_symbol: Symbol, closure_tag_id_layout: Layout<'a>, @@ -8005,7 +8069,7 @@ fn union_lambda_set_to_switch<'a>( assigned: Symbol, hole: &'a Stmt<'a>, ) -> Stmt<'a> { - if lambda_set.is_empty() { + if lambda_set.set.is_empty() { // NOTE this can happen if there is a type error somewhere. Since the lambda set is empty, // there is really nothing we can do here. We generate a runtime error here which allows // code gen to proceed. We then assume that we hit another (more descriptive) error before @@ -8017,11 +8081,12 @@ fn union_lambda_set_to_switch<'a>( let join_point_id = JoinPointId(env.unique_symbol()); - let mut branches = Vec::with_capacity_in(lambda_set.len(), env.arena); + let mut branches = Vec::with_capacity_in(lambda_set.set.len(), env.arena); - for (i, (function_symbol, _)) in lambda_set.iter().enumerate() { + for (i, (function_symbol, _)) in lambda_set.set.iter().enumerate() { let stmt = union_lambda_set_branch( env, + lambda_set, join_point_id, *function_symbol, closure_data_symbol, @@ -8064,6 +8129,7 @@ fn union_lambda_set_to_switch<'a>( #[allow(clippy::too_many_arguments)] fn union_lambda_set_branch<'a>( env: &mut Env<'a, '_>, + lambda_set: LambdaSet<'a>, join_point_id: JoinPointId, function_symbol: Symbol, closure_data_symbol: Symbol, @@ -8079,6 +8145,7 @@ fn union_lambda_set_branch<'a>( union_lambda_set_branch_help( env, function_symbol, + lambda_set, closure_data_symbol, closure_data_layout, argument_symbols_slice, @@ -8093,6 +8160,7 @@ fn union_lambda_set_branch<'a>( fn union_lambda_set_branch_help<'a>( env: &mut Env<'a, '_>, function_symbol: Symbol, + lambda_set: LambdaSet<'a>, closure_data_symbol: Symbol, closure_data_layout: Layout<'a>, argument_symbols_slice: &'a [Symbol], @@ -8105,12 +8173,19 @@ fn union_lambda_set_branch_help<'a>( Layout::Struct(&[]) | Layout::Builtin(Builtin::Int1) | Layout::Builtin(Builtin::Int8) => { (argument_layouts_slice, argument_symbols_slice) } + _ if lambda_set.member_does_not_need_closure_argument(function_symbol) => { + // sometimes unification causes a function that does not itself capture anything + // to still get a lambda set that does store information. We must not pass a closure + // argument in this case + + (argument_layouts_slice, argument_symbols_slice) + } _ => { // extend layouts with the layout of the closure environment let mut argument_layouts = Vec::with_capacity_in(argument_layouts_slice.len() + 1, env.arena); argument_layouts.extend(argument_layouts_slice); - argument_layouts.push(closure_data_layout); + argument_layouts.push(Layout::LambdaSet(lambda_set)); // extend symbols with the symbol of the closure environment let mut argument_symbols = diff --git a/compiler/mono/src/layout.rs b/compiler/mono/src/layout.rs index 3568ce5045..507bcfaf7e 100644 --- a/compiler/mono/src/layout.rs +++ b/compiler/mono/src/layout.rs @@ -189,6 +189,7 @@ pub enum Layout<'a> { /// this is important for closures that capture zero-sized values Struct(&'a [Layout<'a>]), Union(UnionLayout<'a>), + LambdaSet(LambdaSet<'a>), RecursivePointer, } @@ -454,6 +455,17 @@ impl<'a> LambdaSet<'a> { } } + pub fn member_does_not_need_closure_argument(&self, function_symbol: Symbol) -> bool { + match self.layout_for_member(function_symbol) { + ClosureRepresentation::Union { + alphabetic_order_fields, + .. + } => alphabetic_order_fields.is_empty(), + ClosureRepresentation::AlphabeticOrderStruct(fields) => fields.is_empty(), + ClosureRepresentation::Other(_) => false, + } + } + pub fn layout_for_member(&self, function_symbol: Symbol) -> ClosureRepresentation<'a> { debug_assert!( self.set.iter().any(|(s, _)| *s == function_symbol), @@ -531,7 +543,7 @@ impl<'a> LambdaSet<'a> { _ => { let mut arguments = Vec::with_capacity_in(argument_layouts.len() + 1, arena); arguments.extend(argument_layouts); - arguments.push(self.runtime_representation()); + arguments.push(Layout::LambdaSet(*self)); arguments.into_bump_slice() } @@ -587,7 +599,7 @@ impl<'a> LambdaSet<'a> { // this can happen when there is a type error somewhere Ok(LambdaSet { set: &[], - representation: arena.alloc(Layout::Union(UnionLayout::NonRecursive(&[]))), + representation: arena.alloc(Layout::Struct(&[])), }) } _ => panic!("called LambdaSet.from_var on invalid input"), @@ -606,7 +618,9 @@ impl<'a> LambdaSet<'a> { use UnionVariant::*; match variant { Never => Layout::Union(UnionLayout::NonRecursive(&[])), - Unit | UnitWithArguments | BoolUnion { .. } | ByteUnion(_) => { + BoolUnion { .. } => Layout::Builtin(Builtin::Int1), + ByteUnion { .. } => Layout::Builtin(Builtin::Int8), + Unit | UnitWithArguments => { // no useful information to store Layout::Struct(&[]) } @@ -667,7 +681,6 @@ pub enum Builtin<'a> { Float128, Float64, Float32, - Float16, Str, Dict(&'a Layout<'a>, &'a Layout<'a>), Set(&'a Layout<'a>), @@ -826,6 +839,7 @@ impl<'a> Layout<'a> { } } } + LambdaSet(lambda_set) => lambda_set.runtime_representation().safe_to_memcpy(), RecursivePointer => { // We cannot memcpy pointers, because then we would have the same pointer in multiple places! false @@ -890,6 +904,9 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => pointer_size, } } + LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .stack_size_without_alignment(pointer_size), RecursivePointer => pointer_size, } } @@ -919,6 +936,9 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => pointer_size, } } + Layout::LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .alignment_bytes(pointer_size), Layout::Builtin(builtin) => builtin.alignment_bytes(pointer_size), Layout::RecursivePointer => pointer_size, } @@ -929,6 +949,9 @@ impl<'a> Layout<'a> { Layout::Builtin(builtin) => builtin.allocation_alignment_bytes(pointer_size), Layout::Struct(_) => unreachable!("not heap-allocated"), Layout::Union(union_layout) => union_layout.allocation_alignment_bytes(pointer_size), + Layout::LambdaSet(lambda_set) => lambda_set + .runtime_representation() + .allocation_alignment_bytes(pointer_size), Layout::RecursivePointer => unreachable!("should be looked up to get an actual layout"), } } @@ -979,6 +1002,7 @@ impl<'a> Layout<'a> { | NonNullableUnwrapped(_) => true, } } + LambdaSet(lambda_set) => lambda_set.runtime_representation().contains_refcounted(), RecursivePointer => true, } } @@ -1002,6 +1026,7 @@ impl<'a> Layout<'a> { .append(alloc.text("}")) } Union(union_layout) => union_layout.to_doc(alloc, parens), + LambdaSet(lambda_set) => lambda_set.runtime_representation().to_doc(alloc, parens), RecursivePointer => alloc.text("*self"), } } @@ -1093,7 +1118,6 @@ impl<'a> Builtin<'a> { const F128_SIZE: u32 = 16; const F64_SIZE: u32 = std::mem::size_of::() as u32; const F32_SIZE: u32 = std::mem::size_of::() as u32; - const F16_SIZE: u32 = 2; /// Number of machine words in an empty one of these pub const STR_WORDS: u32 = 2; @@ -1123,7 +1147,6 @@ impl<'a> Builtin<'a> { Float128 => Builtin::F128_SIZE, Float64 => Builtin::F64_SIZE, Float32 => Builtin::F32_SIZE, - Float16 => Builtin::F16_SIZE, Str | EmptyStr => Builtin::STR_WORDS * pointer_size, Dict(_, _) | EmptyDict => Builtin::DICT_WORDS * pointer_size, Set(_) | EmptySet => Builtin::SET_WORDS * pointer_size, @@ -1150,7 +1173,6 @@ impl<'a> Builtin<'a> { Float128 => align_of::() as u32, Float64 => align_of::() as u32, Float32 => align_of::() as u32, - Float16 => align_of::() as u32, Dict(_, _) | EmptyDict => pointer_size, Set(_) | EmptySet => pointer_size, // we often treat these as i128 (64-bit systems) @@ -1158,8 +1180,8 @@ impl<'a> Builtin<'a> { // // In webassembly, For that to be safe // they must be aligned to allow such access - List(_) | EmptyList => pointer_size.max(8), - Str | EmptyStr => pointer_size.max(8), + List(_) | EmptyList => pointer_size, + Str | EmptyStr => pointer_size, } } @@ -1168,7 +1190,7 @@ impl<'a> Builtin<'a> { match self { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize | Decimal | Float128 | Float64 - | Float32 | Float16 | EmptyStr | EmptyDict | EmptyList | EmptySet => true, + | Float32 | EmptyStr | EmptyDict | EmptyList | EmptySet => true, Str | Dict(_, _) | Set(_) | List(_) => false, } } @@ -1179,7 +1201,7 @@ impl<'a> Builtin<'a> { match self { Int128 | Int64 | Int32 | Int16 | Int8 | Int1 | Usize | Decimal | Float128 | Float64 - | Float32 | Float16 | EmptyStr | EmptyDict | EmptyList | EmptySet => false, + | Float32 | EmptyStr | EmptyDict | EmptyList | EmptySet => false, List(_) => true, Str | Dict(_, _) | Set(_) => true, @@ -1206,7 +1228,6 @@ impl<'a> Builtin<'a> { Float128 => alloc.text("Float128"), Float64 => alloc.text("Float64"), Float32 => alloc.text("Float32"), - Float16 => alloc.text("Float16"), EmptyStr => alloc.text("EmptyStr"), EmptyList => alloc.text("EmptyList"), @@ -1240,8 +1261,7 @@ impl<'a> Builtin<'a> { | Builtin::Decimal | Builtin::Float128 | Builtin::Float64 - | Builtin::Float32 - | Builtin::Float16 => unreachable!("not heap-allocated"), + | Builtin::Float32 => unreachable!("not heap-allocated"), Builtin::Str => pointer_size, Builtin::Dict(k, v) => k .alignment_bytes(pointer_size) @@ -1360,7 +1380,7 @@ fn layout_from_flat_type<'a>( Func(_, closure_var, _) => { let lambda_set = LambdaSet::from_var(env.arena, env.subs, closure_var, env.ptr_bytes)?; - Ok(lambda_set.runtime_representation()) + Ok(Layout::LambdaSet(lambda_set)) } Record(fields, ext_var) => { // extract any values from the ext_var diff --git a/compiler/reporting/src/error/canonicalize.rs b/compiler/reporting/src/error/canonicalize.rs index 69f1175a6e..13d3406593 100644 --- a/compiler/reporting/src/error/canonicalize.rs +++ b/compiler/reporting/src/error/canonicalize.rs @@ -1192,7 +1192,7 @@ fn not_found<'b>( alloc.reflow(" missing up-top"), ]); - let default_yes = alloc.reflow("these names seem close though:"); + let default_yes = alloc.reflow("Did you mean one of these?"); let to_details = |no_suggestion_details, yes_suggestion_details| { if suggestions.is_empty() { @@ -1240,7 +1240,7 @@ fn module_not_found<'b>( ]); let default_yes = alloc - .reflow("Is there an import missing? Perhaps there is a typo, these names seem close:"); + .reflow("Is there an import missing? Perhaps there is a typo. Did you mean one of these?"); let to_details = |no_suggestion_details, yes_suggestion_details| { if suggestions.is_empty() { diff --git a/compiler/reporting/src/error/type.rs b/compiler/reporting/src/error/type.rs index 115d3f6671..c4cd15d406 100644 --- a/compiler/reporting/src/error/type.rs +++ b/compiler/reporting/src/error/type.rs @@ -1,7 +1,8 @@ use roc_can::expected::{Expected, PExpected}; use roc_collections::all::{Index, MutSet, SendMap}; -use roc_module::ident::{IdentStr, Lowercase, TagName}; +use roc_module::ident::{Ident, IdentStr, Lowercase, TagName}; use roc_module::symbol::Symbol; +use roc_region::all::{Located, Region}; use roc_solve::solve; use roc_types::pretty_print::Parens; use roc_types::types::{Category, ErrorType, PatternCategory, Reason, RecordField, TypeExt}; @@ -10,34 +11,39 @@ use std::path::PathBuf; use crate::report::{Annotation, Report, RocDocAllocator, RocDocBuilder, Severity}; use ven_pretty::DocAllocator; +const DUPLICATE_NAME: &str = "DUPLICATE NAME"; const ADD_ANNOTATIONS: &str = r#"Can more type annotations be added? Type annotations always help me give more specific messages, and I think they could help a lot in this case"#; pub fn type_problem<'b>( alloc: &'b RocDocAllocator<'b>, filename: PathBuf, problem: solve::TypeError, -) -> Report<'b> { +) -> Option> { use solve::TypeError::*; - fn report(title: String, doc: RocDocBuilder<'_>, filename: PathBuf) -> Report<'_> { - Report { + fn report(title: String, doc: RocDocBuilder<'_>, filename: PathBuf) -> Option> { + Some(Report { title, filename, doc, severity: Severity::RuntimeError, - } + }) } match problem { - BadExpr(region, category, found, expected) => { - to_expr_report(alloc, filename, region, category, found, expected) - } - BadPattern(region, category, found, expected) => { - to_pattern_report(alloc, filename, region, category, found, expected) - } - CircularType(region, symbol, overall_type) => { - to_circular_report(alloc, filename, region, symbol, overall_type) - } + BadExpr(region, category, found, expected) => Some(to_expr_report( + alloc, filename, region, category, found, expected, + )), + BadPattern(region, category, found, expected) => Some(to_pattern_report( + alloc, filename, region, category, found, expected, + )), + CircularType(region, symbol, overall_type) => Some(to_circular_report( + alloc, + filename, + region, + symbol, + overall_type, + )), UnexposedLookup(symbol) => { let title = "UNRECOGNIZED NAME".to_string(); let doc = alloc @@ -97,12 +103,40 @@ pub fn type_problem<'b>( report(title, doc, filename) } + SolvedTypeError => None, // Don't re-report cascading errors - see https://github.com/rtfeldman/roc/pull/1711 + + Shadowed(original_region, shadow) => { + let doc = report_shadowing(alloc, original_region, shadow); + let title = DUPLICATE_NAME.to_string(); + + report(title, doc, filename) + } + other => panic!("unhandled bad type: {:?}", other), } } } } +fn report_shadowing<'b>( + alloc: &'b RocDocAllocator<'b>, + original_region: Region, + shadow: Located, +) -> RocDocBuilder<'b> { + let line = r#"Since these types have the same name, it's easy to use the wrong one on accident. Give one of them a new name."#; + + alloc.stack(vec![ + alloc + .text("The ") + .append(alloc.ident(shadow.value)) + .append(alloc.reflow(" name is first defined here:")), + alloc.region(original_region), + alloc.reflow("But then it's defined a second time here:"), + alloc.region(shadow.region), + alloc.reflow(line), + ]) +} + pub fn cyclic_alias<'b>( alloc: &'b RocDocAllocator<'b>, symbol: Symbol, diff --git a/compiler/reporting/src/report.rs b/compiler/reporting/src/report.rs index 28b615efe2..8a9b045f4c 100644 --- a/compiler/reporting/src/report.rs +++ b/compiler/reporting/src/report.rs @@ -128,6 +128,7 @@ impl<'b> Report<'b> { pub struct Palette<'a> { pub primary: &'a str, pub code_block: &'a str, + pub keyword: &'a str, pub variable: &'a str, pub type_variable: &'a str, pub structure: &'a str, @@ -146,6 +147,7 @@ pub struct Palette<'a> { pub const DEFAULT_PALETTE: Palette = Palette { primary: WHITE_CODE, code_block: WHITE_CODE, + keyword: GREEN_CODE, variable: BLUE_CODE, type_variable: YELLOW_CODE, structure: GREEN_CODE, @@ -810,6 +812,9 @@ where Symbol => { self.write_str(self.palette.variable)?; } + Keyword => { + self.write_str(self.palette.keyword)?; + } GutterBar => { self.write_str(self.palette.gutter_bar)?; } @@ -837,7 +842,7 @@ where ParserSuggestion => { self.write_str(self.palette.parser_suggestion)?; } - TypeBlock | GlobalTag | PrivateTag | RecordField | Keyword => { /* nothing yet */ } + TypeBlock | GlobalTag | PrivateTag | RecordField => { /* nothing yet */ } } self.style_stack.push(*annotation); Ok(()) @@ -851,11 +856,11 @@ where Some(annotation) => match annotation { Emphasized | Url | TypeVariable | Alias | Symbol | BinOp | Error | GutterBar | Typo | TypoSuggestion | ParserSuggestion | Structure | CodeBlock | PlainText - | LineNumber | Tip | Module | Header => { + | LineNumber | Tip | Module | Header | Keyword => { self.write_str(RESET_CODE)?; } - TypeBlock | GlobalTag | PrivateTag | RecordField | Keyword => { /* nothing yet */ } + TypeBlock | GlobalTag | PrivateTag | RecordField => { /* nothing yet */ } }, } Ok(()) diff --git a/compiler/reporting/tests/test_reporting.rs b/compiler/reporting/tests/test_reporting.rs index 7575aca5b0..14fb7279ad 100644 --- a/compiler/reporting/tests/test_reporting.rs +++ b/compiler/reporting/tests/test_reporting.rs @@ -154,8 +154,9 @@ mod test_reporting { } for problem in type_problems { - let report = type_problem(&alloc, filename.clone(), problem.clone()); - reports.push(report); + if let Some(report) = type_problem(&alloc, filename.clone(), problem.clone()) { + reports.push(report); + } } for problem in mono_problems { @@ -541,7 +542,7 @@ mod test_reporting { 8│ 4 -> bar baz "yay" ^^^ - these names seem close though: + Did you mean one of these? baz Nat @@ -739,7 +740,7 @@ mod test_reporting { 3│ theAdmin ^^^^^^^^ - these names seem close though: + Did you mean one of these? Decimal Dec @@ -1491,7 +1492,7 @@ mod test_reporting { 2│ { foo: 2 } -> foo ^^^ - these names seem close though: + Did you mean one of these? Bool U8 @@ -1947,7 +1948,7 @@ mod test_reporting { 2│ f = \_ -> ok 4 ^^ - these names seem close though: + Did you mean one of these? U8 f @@ -3634,8 +3635,8 @@ mod test_reporting { 1│ Foo.test ^^^^^^^^ - Is there an import missing? Perhaps there is a typo, these names seem - close: + Is there an import missing? Perhaps there is a typo. Did you mean one + of these? Bool Num @@ -5797,7 +5798,7 @@ mod test_reporting { 1│ [ "foo", bar("") ] ^^^ - these names seem close though: + Did you mean one of these? Nat Str diff --git a/compiler/test_gen/src/gen_list.rs b/compiler/test_gen/src/gen_list.rs index 1fd096f0a6..758603f2c1 100644 --- a/compiler/test_gen/src/gen_list.rs +++ b/compiler/test_gen/src/gen_list.rs @@ -2017,3 +2017,45 @@ fn lists_with_incompatible_type_param_in_if() { RocStr ); } + +#[test] +fn map_with_index_multi_record() { + // see https://github.com/rtfeldman/roc/issues/1700 + assert_evals_to!( + indoc!( + r#" + List.mapWithIndex [ { x: {}, y: {} } ] \_, _ -> {} + "# + ), + RocList::from_slice(&[((), ())]), + RocList<((), ())> + ); +} + +#[test] +fn empty_list_of_function_type() { + // see https://github.com/rtfeldman/roc/issues/1732 + assert_evals_to!( + indoc!( + r#" + myList : List (Str -> Str) + myList = [] + + myClosure : Str -> Str + myClosure = \_ -> "bar" + + choose = + if False then + myList + else + [ myClosure ] + + when List.get choose 0 is + Ok f -> f "foo" + Err _ -> "bad!" + "# + ), + RocStr::from_slice(b"bar"), + RocStr + ); +} diff --git a/compiler/test_gen/src/gen_num.rs b/compiler/test_gen/src/gen_num.rs index 29bb479855..9d9d9e1390 100644 --- a/compiler/test_gen/src/gen_num.rs +++ b/compiler/test_gen/src/gen_num.rs @@ -1778,4 +1778,48 @@ mod gen_num { u32 ); } + + #[test] + fn when_on_i32() { + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + x : I32 + x = 0 + + main : I32 + main = + when x is + 0 -> 42 + _ -> -1 + "# + ), + 42, + i32 + ); + } + + #[test] + fn when_on_i16() { + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + x : I16 + x = 0 + + main : I16 + main = + when x is + 0 -> 42 + _ -> -1 + "# + ), + 42, + i16 + ); + } } diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs index 9a43cb8a78..d183b95d0b 100644 --- a/compiler/test_gen/src/gen_primitives.rs +++ b/compiler/test_gen/src/gen_primitives.rs @@ -2531,6 +2531,8 @@ fn pattern_match_unit_tag() { ); } +// see for why this is disabled on wasm32 https://github.com/rtfeldman/roc/issues/1687 +#[cfg(not(feature = "wasm-cli-run"))] #[test] fn mirror_llvm_alignment_padding() { // see https://github.com/rtfeldman/roc/issues/1569 @@ -2616,7 +2618,8 @@ fn lambda_set_struct_byte() { r = Red p1 = (\u -> r == u) - oneOfResult = List.map [p1, p1] (\p -> p Green) + foobarbaz = (\p -> p Green) + oneOfResult = List.map [p1, p1] foobarbaz when oneOfResult is _ -> 32 @@ -2779,3 +2782,127 @@ fn value_not_exposed_hits_panic() { i64 ); } + +#[test] +fn mix_function_and_closure() { + // see https://github.com/rtfeldman/roc/pull/1706 + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + # foo does not capture any variables + # but through unification will get a lambda set that does store information + # we must handle that correctly + foo = \x -> x + + bar = \y -> \_ -> y + + main : Str + main = + (if 1 == 1 then foo else (bar "nope nope nope")) "hello world" + "# + ), + RocStr::from_slice(b"hello world"), + RocStr + ); +} + +#[test] +fn mix_function_and_closure_level_of_indirection() { + // see https://github.com/rtfeldman/roc/pull/1706 + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + foo = \x -> x + + bar = \y -> \_ -> y + + f = (if 1 == 1 then foo else (bar "nope nope nope")) + + main : Str + main = + f "hello world" + "# + ), + RocStr::from_slice(b"hello world"), + RocStr + ); +} + +#[test] +fn do_pass_bool_byte_closure_layout() { + // see https://github.com/rtfeldman/roc/pull/1706 + // the distinction is actually important, dropping that info means some functions just get + // skipped + assert_evals_to!( + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + ## PARSER + + Parser a : List U8 -> List [Pair a (List U8)] + + + ## ANY + + # If succcessful, the any parser consumes one character + + any: Parser U8 + any = \inp -> + when List.first inp is + Ok u -> [Pair u (List.drop inp 1)] + _ -> [ ] + + + + ## SATISFY + + satisfy : (U8 -> Bool) -> Parser U8 + satisfy = \predicate -> + \input -> + walker = \(Pair u rest), accum -> + if predicate u then + Stop [ Pair u rest ] + + else + Stop accum + + List.walkUntil (any input) walker [] + + + + oneOf : List (Parser a) -> Parser a + oneOf = \parserList -> + \input -> + walker = \p, accum -> + output = p input + if List.len output == 1 then + Stop output + + else + Continue accum + + List.walkUntil parserList walker [] + + + satisfyA = satisfy (\u -> u == 97) # recognize 97 + satisfyB = satisfy (\u -> u == 98) # recognize 98 + + test1 = if List.len ((oneOf [satisfyA, satisfyB]) [97, 98, 99, 100] ) == 1 then "PASS" else "FAIL" + test2 = if List.len ((oneOf [satisfyA, satisfyB]) [98, 99, 100, 97] ) == 1 then "PASS" else "FAIL" + test3 = if List.len ((oneOf [satisfyB , satisfyA]) [98, 99, 100, 97] ) == 1 then "PASS" else "FAIL" + test4 = if List.len ((oneOf [satisfyA, satisfyB]) [99, 100, 101] ) == 0 then "PASS" else "FAIL" + + + main : Str + main = [test1, test2, test3, test4] |> Str.joinWith ", " + "# + ), + RocStr::from_slice(b"PASS, PASS, PASS, PASS"), + RocStr + ); +} diff --git a/compiler/test_gen/src/gen_str.rs b/compiler/test_gen/src/gen_str.rs index 2b5bdb45ad..292946403e 100644 --- a/compiler/test_gen/src/gen_str.rs +++ b/compiler/test_gen/src/gen_str.rs @@ -949,3 +949,31 @@ fn str_from_utf8_range_count_too_high_for_start() { RocStr ); } + +#[test] +fn str_repeat_small() { + assert_evals_to!( + indoc!(r#"Str.repeat "Roc" 3"#), + RocStr::from("RocRocRoc"), + RocStr + ); +} + +#[test] +fn str_repeat_big() { + assert_evals_to!( + indoc!(r#"Str.repeat "more than 16 characters" 2"#), + RocStr::from("more than 16 charactersmore than 16 characters"), + RocStr + ); +} + +#[test] +fn str_repeat_empty_string() { + assert_evals_to!(indoc!(r#"Str.repeat "" 3"#), RocStr::from(""), RocStr); +} + +#[test] +fn str_repeat_zero_times() { + assert_evals_to!(indoc!(r#"Str.repeat "Roc" 0"#), RocStr::from(""), RocStr); +} diff --git a/compiler/test_gen/src/helpers/eval.rs b/compiler/test_gen/src/helpers/eval.rs index dc4c377c84..be01cdc7ab 100644 --- a/compiler/test_gen/src/helpers/eval.rs +++ b/compiler/test_gen/src/helpers/eval.rs @@ -143,12 +143,13 @@ fn create_llvm_module<'a>( } for problem in type_problems { - let report = type_problem(&alloc, module_path.clone(), problem); - let mut buf = String::new(); + if let Some(report) = type_problem(&alloc, module_path.clone(), problem) { + let mut buf = String::new(); - report.render_color_terminal(&mut buf, &alloc, &palette); + report.render_color_terminal(&mut buf, &alloc, &palette); - lines.push(buf); + lines.push(buf); + } } for problem in mono_problems { diff --git a/compiler/test_mono/generated/empty_list_of_function_type.txt b/compiler/test_mono/generated/empty_list_of_function_type.txt new file mode 100644 index 0000000000..89aa3d309b --- /dev/null +++ b/compiler/test_mono/generated/empty_list_of_function_type.txt @@ -0,0 +1,43 @@ +procedure List.3 (#Attr.2, #Attr.3): + let Test.20 = lowlevel ListLen #Attr.2; + let Test.17 = lowlevel NumLt #Attr.3 Test.20; + if Test.17 then + let Test.19 = lowlevel ListGetUnsafe #Attr.2 #Attr.3; + let Test.18 = Ok Test.19; + ret Test.18; + else + let Test.16 = Struct {}; + let Test.15 = Err Test.16; + ret Test.15; + +procedure Test.2 (Test.6): + let Test.24 = "bar"; + ret Test.24; + +procedure Test.0 (): + let Test.1 = Array []; + joinpoint Test.22 Test.3: + let Test.14 = 0i64; + let Test.7 = CallByName List.3 Test.3 Test.14; + dec Test.3; + let Test.11 = 1i64; + let Test.12 = GetTagId Test.7; + let Test.13 = lowlevel Eq Test.11 Test.12; + if Test.13 then + let Test.5 = UnionAtIndex (Id 1) (Index 0) Test.7; + let Test.9 = "foo"; + let Test.8 = CallByName Test.2 Test.9; + dec Test.9; + ret Test.8; + else + let Test.10 = "bad!"; + ret Test.10; + in + let Test.25 = false; + if Test.25 then + jump Test.22 Test.1; + else + dec Test.1; + let Test.23 = Struct {}; + let Test.21 = Array [Test.23]; + jump Test.22 Test.21; diff --git a/compiler/test_mono/generated/quicksort_help.txt b/compiler/test_mono/generated/quicksort_help.txt index 8435559668..79af94f700 100644 --- a/compiler/test_mono/generated/quicksort_help.txt +++ b/compiler/test_mono/generated/quicksort_help.txt @@ -10,7 +10,7 @@ procedure Num.27 (#Attr.2, #Attr.3): let Test.26 = lowlevel NumLt #Attr.2 #Attr.3; ret Test.26; -procedure Test.1 (Test.29, Test.30, Test.31): +procedure Test.1 (Test.27, Test.28, Test.29): joinpoint Test.12 Test.2 Test.3 Test.4: let Test.14 = CallByName Num.27 Test.3 Test.4; if Test.14 then @@ -29,7 +29,7 @@ procedure Test.1 (Test.29, Test.30, Test.31): else ret Test.2; in - jump Test.12 Test.29 Test.30 Test.31; + jump Test.12 Test.27 Test.28 Test.29; procedure Test.0 (): let Test.9 = Array []; diff --git a/compiler/test_mono/generated/somehow_drops_definitions.txt b/compiler/test_mono/generated/somehow_drops_definitions.txt index 15ddbd26c6..2e7a497919 100644 --- a/compiler/test_mono/generated/somehow_drops_definitions.txt +++ b/compiler/test_mono/generated/somehow_drops_definitions.txt @@ -1,43 +1,53 @@ procedure Num.24 (#Attr.2, #Attr.3): - let Test.24 = lowlevel NumAdd #Attr.2 #Attr.3; - ret Test.24; + let Test.27 = lowlevel NumAdd #Attr.2 #Attr.3; + ret Test.27; procedure Num.26 (#Attr.2, #Attr.3): - let Test.19 = lowlevel NumMul #Attr.2 #Attr.3; - ret Test.19; - -procedure Test.1 (): - let Test.25 = 1i64; - ret Test.25; - -procedure Test.2 (): - let Test.20 = 2i64; - ret Test.20; - -procedure Test.3 (Test.6): - let Test.23 = CallByName Test.1; - let Test.22 = CallByName Num.24 Test.6 Test.23; + let Test.22 = lowlevel NumMul #Attr.2 #Attr.3; ret Test.22; +procedure Test.1 (): + let Test.28 = 1i64; + ret Test.28; + +procedure Test.2 (): + let Test.23 = 2i64; + ret Test.23; + +procedure Test.3 (Test.6): + let Test.26 = CallByName Test.1; + let Test.25 = CallByName Num.24 Test.6 Test.26; + ret Test.25; + procedure Test.4 (Test.7): - let Test.18 = CallByName Test.2; - let Test.17 = CallByName Num.26 Test.7 Test.18; - ret Test.17; + let Test.21 = CallByName Test.2; + let Test.20 = CallByName Num.26 Test.7 Test.21; + ret Test.20; procedure Test.5 (Test.8, Test.9): - let Test.14 = CallByName Test.3 Test.9; - ret Test.14; + joinpoint Test.15 Test.14: + ret Test.14; + in + switch Test.8: + case 0: + let Test.16 = CallByName Test.3 Test.9; + jump Test.15 Test.16; + + default: + let Test.17 = CallByName Test.4 Test.9; + jump Test.15 Test.17; + procedure Test.0 (): - joinpoint Test.16 Test.12: + joinpoint Test.19 Test.12: let Test.13 = 42i64; let Test.11 = CallByName Test.5 Test.12 Test.13; ret Test.11; in - let Test.21 = true; - if Test.21 then - let Test.3 = Struct {}; - jump Test.16 Test.3; + let Test.24 = true; + if Test.24 then + let Test.3 = false; + jump Test.19 Test.3; else - let Test.4 = Struct {}; - jump Test.16 Test.4; + let Test.4 = true; + jump Test.19 Test.4; diff --git a/compiler/test_mono/src/lib.rs b/compiler/test_mono/src/lib.rs index 2d05f9cd16..cbcdbed066 100644 --- a/compiler/test_mono/src/lib.rs +++ b/compiler/test_mono/src/lib.rs @@ -1084,6 +1084,33 @@ fn specialize_lowlevel() { ) } +#[mono_test] +fn empty_list_of_function_type() { + // see https://github.com/rtfeldman/roc/issues/1732 + indoc!( + r#" + app "test" provides [ main ] to "./platform" + + main = + myList : List (Str -> Str) + myList = [] + + myClosure : Str -> Str + myClosure = \_ -> "bar" + + choose = + if False then + myList + else + [ myClosure ] + + when List.get choose 0 is + Ok f -> f "foo" + Err _ -> "bad!" + "# + ) +} + // #[ignore] // #[mono_test] // fn static_str_closure() { diff --git a/examples/.gitignore b/examples/.gitignore index 874bbd86e5..c0e522cc76 100644 --- a/examples/.gitignore +++ b/examples/.gitignore @@ -4,3 +4,7 @@ app libhost.a roc_app.ll roc_app.bc +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file diff --git a/examples/benchmarks/platform/host.zig b/examples/benchmarks/platform/host.zig index 503a66ed2d..aa339ec306 100644 --- a/examples/benchmarks/platform/host.zig +++ b/examples/benchmarks/platform/host.zig @@ -23,16 +23,19 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed([*]u8) void; +extern fn roc__mainForHost_1_exposed_generic([*]u8) void; extern fn roc__mainForHost_size() i64; extern fn roc__mainForHost_1_Fx_caller(*const u8, [*]u8, [*]u8) void; extern fn roc__mainForHost_1_Fx_size() i64; extern fn roc__mainForHost_1_Fx_result_size() i64; -const Align = extern struct { a: usize, b: usize }; -extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; -extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; -extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +const Align = 2 * @alignOf(usize); +extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void; +extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -53,7 +56,7 @@ export fn roc_realloc(c_ptr: *c_void, new_size: usize, old_size: usize, alignmen stdout.print("realloc: {d} (alignment {d}, old_size {d})\n", .{ c_ptr, alignment, old_size }) catch unreachable; } - return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); + return realloc(@alignCast(Align, @ptrCast([*]u8, c_ptr)), new_size); } export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { @@ -62,7 +65,7 @@ export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { stdout.print("dealloc: {d} (alignment {d})\n", .{ c_ptr, alignment }) catch unreachable; } - free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); + free(@alignCast(Align, @ptrCast([*]u8, c_ptr))); } export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { @@ -74,9 +77,17 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; -pub fn main() u8 { +pub export fn main() callconv(.C) u8 { const allocator = std.heap.page_allocator; const size = @intCast(usize, roc__mainForHost_size()); @@ -90,23 +101,11 @@ pub fn main() u8 { var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; - roc__mainForHost_1_exposed(output); + roc__mainForHost_1_exposed_generic(output); - const flag = @ptrCast(*u64, @alignCast(@alignOf(u64), output)).*; + const closure_data_pointer = @ptrCast([*]u8, output); - if (flag == 0) { - // all is well - const closure_data_pointer = @ptrCast([*]u8, output[@sizeOf(u64)..size]); - - call_the_closure(closure_data_pointer); - } else { - const ptr = @ptrCast(*u32, output + @sizeOf(u64)); - const msg = @intToPtr([*:0]const u8, ptr.*); - const stderr = std.io.getStdErr().writer(); - stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; - - return 0; - } + call_the_closure(closure_data_pointer); var ts2: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; diff --git a/examples/cli/platform/src/lib.rs b/examples/cli/platform/src/lib.rs index 2a98179f8e..2b24da5ff7 100644 --- a/examples/cli/platform/src/lib.rs +++ b/examples/cli/platform/src/lib.rs @@ -4,7 +4,7 @@ use core::alloc::Layout; use core::ffi::c_void; use core::mem::MaybeUninit; use libc; -use roc_std::{RocCallResult, RocStr}; +use roc_std::RocStr; use std::ffi::CStr; use std::os::raw::c_char; @@ -70,23 +70,11 @@ pub fn rust_main() -> isize { roc_main(buffer); - let output = buffer as *mut RocCallResult<()>; + let result = call_the_closure(buffer); - match (&*output).into() { - Ok(()) => { - let closure_data_ptr = buffer.offset(8); - let result = call_the_closure(closure_data_ptr as *const u8); + std::alloc::dealloc(buffer, layout); - std::alloc::dealloc(buffer, layout); - - result - } - Err(msg) => { - std::alloc::dealloc(buffer, layout); - - panic!("Roc failed with message: {}", msg); - } - } + result }; // Exit code @@ -105,15 +93,9 @@ unsafe fn call_the_closure(closure_data_ptr: *const u8) -> i64 { buffer as *mut u8, ); - let output = &*(buffer as *mut RocCallResult<()>); + std::alloc::dealloc(buffer, layout); - match output.into() { - Ok(_) => { - std::alloc::dealloc(buffer, layout); - 0 - } - Err(e) => panic!("failed with {}", e), - } + 0 } #[no_mangle] diff --git a/examples/effect/Main.roc b/examples/effect/Main.roc index 9e3dd5f506..15eb6ef6e8 100644 --- a/examples/effect/Main.roc +++ b/examples/effect/Main.roc @@ -5,4 +5,7 @@ app "effect-example" main : Effect.Effect {} main = - Effect.after Effect.getLine \lineThisThing -> Effect.putLine lineThisThing + Effect.after (Effect.getLine) \line -> + Effect.after (Effect.putLine "You entered: \(line)") \{} -> + Effect.after (Effect.putLine "It is known") \{} -> + Effect.always {} diff --git a/examples/effect/thing/platform-dir/host.zig b/examples/effect/thing/platform-dir/host.zig index 063d28973f..49fad7a204 100644 --- a/examples/effect/thing/platform-dir/host.zig +++ b/examples/effect/thing/platform-dir/host.zig @@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64; extern fn malloc(size: usize) callconv(.C) ?*c_void; extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { return malloc(size); @@ -52,18 +54,29 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const Unit = extern struct {}; pub export fn main() u8 { + const allocator = std.heap.page_allocator; + const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - const size = @intCast(usize, roc__mainForHost_size()); - const raw_output = std.heap.c_allocator.alloc(u8, size) catch unreachable; + // NOTE the return size can be zero, which will segfault. Always allocate at least 8 bytes + const size = std.math.max(8, @intCast(usize, roc__mainForHost_size())); + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } var ts1: std.os.timespec = undefined; @@ -71,21 +84,7 @@ pub export fn main() u8 { roc__mainForHost_1_exposed(output); - const elements = @ptrCast([*]u64, @alignCast(8, output)); - - var flag = elements[0]; - - if (flag == 0) { - // all is well - const closure_data_pointer = @ptrCast([*]u8, output[8..size]); - - call_the_closure(closure_data_pointer); - } else { - const msg = @intToPtr([*:0]const u8, elements[1]); - stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; - - return 0; - } + call_the_closure(output); var ts2: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; @@ -102,27 +101,20 @@ fn to_seconds(tms: std.os.timespec) f64 { } fn call_the_closure(closure_data_pointer: [*]u8) void { + const allocator = std.heap.page_allocator; + const size = roc__mainForHost_1_Fx_result_size(); - const raw_output = std.heap.c_allocator.alloc(u8, @intCast(usize, size)) catch unreachable; + const raw_output = allocator.allocAdvanced(u8, @alignOf(u64), @intCast(usize, size), .at_least) catch unreachable; var output = @ptrCast([*]u8, raw_output); defer { - std.heap.c_allocator.free(raw_output); + allocator.free(raw_output); } const flags: u8 = 0; - roc__mainForHost_1_Fx_caller(&flags, closure_data_pointer, output); - const elements = @ptrCast([*]u64, @alignCast(8, output)); - - var flag = elements[0]; - - if (flag == 0) { - return; - } else { - unreachable; - } + return; } pub export fn roc_fx_getLine() str.RocStr { diff --git a/examples/fib/.gitignore b/examples/fib/.gitignore new file mode 100644 index 0000000000..41d1223f94 --- /dev/null +++ b/examples/fib/.gitignore @@ -0,0 +1,2 @@ +add +fib diff --git a/examples/fib/Fib.roc b/examples/fib/Fib.roc new file mode 100644 index 0000000000..e5098a023b --- /dev/null +++ b/examples/fib/Fib.roc @@ -0,0 +1,15 @@ +app "fib" + packages { base: "platform" } + imports [] + provides [ main ] to base + +main = \n -> fib n 0 1 + + +# the clever implementation requires join points +fib = \n, a, b -> + if n == 0 then + a + + else + fib (n - 1) b (a + b) diff --git a/examples/fib/platform/Package-Config.roc b/examples/fib/platform/Package-Config.roc new file mode 100644 index 0000000000..0990940738 --- /dev/null +++ b/examples/fib/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform examples/add + requires {}{ main : I64 -> I64 } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : I64 -> I64 +mainForHost = \a -> main a diff --git a/examples/fib/platform/host.zig b/examples/fib/platform/host.zig new file mode 100644 index 0000000000..111a84c27f --- /dev/null +++ b/examples/fib/platform/host.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +const mem = std.mem; +const Allocator = mem.Allocator; + +// NOTE the LLVM backend expects this signature +// extern fn roc__mainForHost_1_exposed(i64, *i64) void; +extern fn roc__mainForHost_1_exposed(i64) i64; + +const Align = extern struct { a: usize, b: usize }; +extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +const DEBUG: bool = false; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + if (DEBUG) { + var ptr = malloc(size); + const stdout = std.io.getStdOut().writer(); + stdout.print("alloc: {d} (alignment {d}, size {d})\n", .{ ptr, alignment, size }) catch unreachable; + return ptr; + } else { + return malloc(size); + } +} + +export fn roc_realloc(c_ptr: *c_void, new_size: usize, old_size: usize, alignment: u32) callconv(.C) ?*c_void { + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("realloc: {d} (alignment {d}, old_size {d})\n", .{ c_ptr, alignment, old_size }) catch unreachable; + } + + return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + if (DEBUG) { + const stdout = std.io.getStdOut().writer(); + stdout.print("dealloc: {d} (alignment {d})\n", .{ c_ptr, alignment }) catch unreachable; + } + + free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); +} + +export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { + _ = tag_id; + + const stderr = std.io.getStdErr().writer(); + const msg = @ptrCast([*:0]const u8, c_ptr); + stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; + std.process.exit(0); +} + +pub export fn main() u8 { + const stdout = std.io.getStdOut().writer(); + const stderr = std.io.getStdErr().writer(); + + // start time + var ts1: std.os.timespec = undefined; + std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; + + const result = roc__mainForHost_1_exposed(10); + + stdout.print("{d}\n", .{result}) catch unreachable; + + // end time + var ts2: std.os.timespec = undefined; + std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts2) catch unreachable; + + const delta = to_seconds(ts2) - to_seconds(ts1); + + stderr.print("runtime: {d:.3}ms\n", .{delta * 1000}) catch unreachable; + + return 0; +} + +fn to_seconds(tms: std.os.timespec) f64 { + return @intToFloat(f64, tms.tv_sec) + (@intToFloat(f64, tms.tv_nsec) / 1_000_000_000.0); +} diff --git a/examples/hello-rust/platform/host.c b/examples/hello-rust/platform/host.c index 0378c69589..9b91965724 100644 --- a/examples/hello-rust/platform/host.c +++ b/examples/hello-rust/platform/host.c @@ -1,7 +1,12 @@ #include +#include extern int rust_main(); -int main() { - return rust_main(); +int main() { return rust_main(); } + +void *roc_memcpy(void *dest, const void *src, size_t n) { + return memcpy(dest, src, n); } + +void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); } \ No newline at end of file diff --git a/examples/hello-rust/platform/src/lib.rs b/examples/hello-rust/platform/src/lib.rs index 8726cab399..6a78b4db0c 100644 --- a/examples/hello-rust/platform/src/lib.rs +++ b/examples/hello-rust/platform/src/lib.rs @@ -3,12 +3,12 @@ use core::ffi::c_void; use core::mem::MaybeUninit; use libc::c_char; -use roc_std::{RocCallResult, RocStr}; +use roc_std::RocStr; use std::ffi::CStr; extern "C" { #[link_name = "roc__mainForHost_1_exposed"] - fn roc_main(output: *mut RocCallResult) -> (); + fn roc_main() -> RocStr; } #[no_mangle] @@ -46,25 +46,14 @@ pub unsafe fn roc_panic(c_ptr: *mut c_void, tag_id: u32) { #[no_mangle] pub fn rust_main() -> isize { - let mut call_result: MaybeUninit> = MaybeUninit::uninit(); - unsafe { - roc_main(call_result.as_mut_ptr()); + let roc_str = roc_main(); - let output = call_result.assume_init(); + let len = roc_str.len(); + let str_bytes = roc_str.get_bytes() as *const libc::c_void; - match output.into() { - Ok(roc_str) => { - let len = roc_str.len(); - let str_bytes = roc_str.get_bytes() as *const libc::c_void; - - if libc::write(1, str_bytes, len) < 0 { - panic!("Writing to stdout failed!"); - } - } - Err(msg) => { - panic!("Roc failed with message: {}", msg); - } + if libc::write(1, str_bytes, len) < 0 { + panic!("Writing to stdout failed!"); } } diff --git a/examples/hello-web/.gitignore b/examples/hello-web/.gitignore new file mode 100644 index 0000000000..227b499154 --- /dev/null +++ b/examples/hello-web/.gitignore @@ -0,0 +1,2 @@ +hello-web +*.wat diff --git a/examples/hello-web/Hello.roc b/examples/hello-web/Hello.roc new file mode 100644 index 0000000000..49f05ceacf --- /dev/null +++ b/examples/hello-web/Hello.roc @@ -0,0 +1,12 @@ +app "hello-web" + packages { base: "platform" } + imports [] + provides [ main ] to base + +greeting = + hi = "Hello" + name = "World" + + "\(hi), \(name)!" + +main = greeting diff --git a/examples/hello-web/README.md b/examples/hello-web/README.md new file mode 100644 index 0000000000..f0beccdb6b --- /dev/null +++ b/examples/hello-web/README.md @@ -0,0 +1,49 @@ +# Hello, World! + +To run, go to the project home directory and run: + +```bash +$ cargo run -- build --backend=wasm32 examples/hello-web/Hello.roc +``` + +Then `cd` into the example directory and run any web server that can handle WebAssembly. +For example with `http-server`: + +```bash +cd examples/hello-web +npm install -g http-server +http-server +``` + +Now open your browser at http://localhost:8080 + +## Design Notes + +This demonstrates the basic design of hosts: Roc code gets compiled into a pure +function (in this case, a thunk that always returns `"Hello, World!"`) and +then the host calls that function. Fundamentally, that's the whole idea! The host +might not even have a `main` - it could be a library, a plugin, anything. +Everything else is built on this basic "hosts calling linked pure functions" design. + +For example, things get more interesting when the compiled Roc function returns +a `Task` - that is, a tagged union data structure containing function pointers +to callback closures. This lets the Roc pure function describe arbitrary +chainable effects, which the host can interpret to perform I/O as requested by +the Roc program. (The tagged union `Task` would have a variant for each supported +I/O operation.) + +In this trivial example, it's very easy to line up the API between the host and +the Roc program. In a more involved host, this would be much trickier - especially +if the API were changing frequently during development. + +The idea there is to have a first-class concept of "glue code" which host authors +can write (it would be plain Roc code, but with some extra keywords that aren't +available in normal modules - kinda like `port module` in Elm), and which +describe both the Roc-host/C boundary as well as the Roc-host/Roc-app boundary. +Roc application authors only care about the Roc-host/Roc-app portion, and the +host author only cares about the Roc-host/C boundary when implementing the host. + +Using this glue code, the Roc compiler can generate C header files describing the +boundary. This not only gets us host compatibility with C compilers, but also +Rust FFI for free, because [`rust-bindgen`](https://github.com/rust-lang/rust-bindgen) +generates correct Rust FFI bindings from C headers. diff --git a/examples/hello-web/index.html b/examples/hello-web/index.html new file mode 100644 index 0000000000..87ca0302cb --- /dev/null +++ b/examples/hello-web/index.html @@ -0,0 +1,12 @@ + + +
+ + + + diff --git a/examples/hello-web/platform/Package-Config.roc b/examples/hello-web/platform/Package-Config.roc new file mode 100644 index 0000000000..377d5c0994 --- /dev/null +++ b/examples/hello-web/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform examples/hello-world + requires {}{ main : Str } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : Str +mainForHost = main diff --git a/examples/hello-web/platform/host.js b/examples/hello-web/platform/host.js new file mode 100644 index 0000000000..a90ff4187b --- /dev/null +++ b/examples/hello-web/platform/host.js @@ -0,0 +1,57 @@ +async function roc_web_platform_run(wasm_filename, callback) { + const decoder = new TextDecoder(); + let memory_bytes; + let exit_code; + + function js_display_roc_string(str_bytes, str_len) { + const utf8_bytes = memory_bytes.subarray(str_bytes, str_bytes + str_len); + const js_string = decoder.decode(utf8_bytes); + callback(js_string); + } + + const importObj = { + wasi_snapshot_preview1: { + proc_exit: (code) => { + if (code !== 0) { + console.error(`Exited with code ${code}`); + } + exit_code = code; + }, + roc_panic: (_pointer, _tag_id) => { + throw 'Roc panicked!'; + } + }, + env: { + js_display_roc_string, + }, + }; + + let wasm; + + const response = await fetch(wasm_filename); + + if (WebAssembly.instantiateStreaming) { + // streaming API has better performance if available + wasm = await WebAssembly.instantiateStreaming(response, importObj); + } else { + const module_bytes = await response.arrayBuffer(); + wasm = await WebAssembly.instantiate(module_bytes, importObj); + } + + memory_bytes = new Uint8Array(wasm.instance.exports.memory.buffer); + + try { + wasm.instance.exports._start(); + } catch (e) { + const is_ok = e.message === "unreachable" && exit_code === 0; + if (!is_ok) { + console.error(e); + } + } +} + +if (typeof module !== 'undefined') { + module.exports = { + roc_web_platform_run, + }; +} diff --git a/examples/hello-web/platform/host.zig b/examples/hello-web/platform/host.zig new file mode 100644 index 0000000000..bfd519e9de --- /dev/null +++ b/examples/hello-web/platform/host.zig @@ -0,0 +1,68 @@ +const std = @import("std"); +const str = @import("str"); +const RocStr = str.RocStr; +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +const Align = extern struct { a: usize, b: usize }; +extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + _ = alignment; + + return malloc(size); +} + +export fn roc_realloc(c_ptr: *c_void, old_size: usize, new_size: usize, alignment: u32) callconv(.C) ?*c_void { + _ = old_size; + _ = alignment; + + return realloc(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + _ = alignment; + + free(@alignCast(@alignOf(Align), @ptrCast([*]u8, c_ptr))); +} + +// NOTE roc_panic is provided in the JS file, so it can throw an exception + +const mem = std.mem; +const Allocator = mem.Allocator; + +extern fn roc__mainForHost_1_exposed(*RocStr) void; + +const Unit = extern struct {}; + +extern fn js_display_roc_string(str_bytes: ?[*]u8, str_len: usize) void; + +pub fn main() u8 { + // actually call roc to populate the callresult + var callresult = RocStr.empty(); + roc__mainForHost_1_exposed(&callresult); + + // display the result using JavaScript + js_display_roc_string(callresult.asU8ptr(), callresult.len()); + + callresult.deinit(); + + return 0; +} diff --git a/examples/hello-web/test-node.js b/examples/hello-web/test-node.js new file mode 100644 index 0000000000..ce1eed5d6e --- /dev/null +++ b/examples/hello-web/test-node.js @@ -0,0 +1,25 @@ +/** + * Node.js test file for hello-web example + * We are not running this in CI currently, and Node.js is not a Roc dependency. + * But if you happen to have it, you can run this. + */ + +// Node doesn't have the fetch API +const fs = require("fs/promises"); +global.fetch = (filename) => + fs.readFile(filename).then((buffer) => ({ + arrayBuffer() { + return buffer; + }, + })); + +const { roc_web_platform_run } = require("./platform/host"); + +roc_web_platform_run("./hello-world.wasm", (string_from_roc) => { + const expected = "Hello, World!"; + if (string_from_roc !== expected) { + console.error(`Expected "${expected}", but got "${string_from_roc}"`); + process.exit(1); + } + console.log("OK"); +}); diff --git a/examples/hello-world/platform/host.c b/examples/hello-world/platform/host.c index f968d0c763..99ada123fe 100644 --- a/examples/hello-world/platform/host.c +++ b/examples/hello-world/platform/host.c @@ -1,89 +1,81 @@ -#include -#include -#include #include -#include #include +#include +#include +#include +#include -void* roc_alloc(size_t size, unsigned int alignment) { - return malloc(size); +void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); } + +void* roc_realloc(void* ptr, size_t old_size, size_t new_size, + unsigned int alignment) { + return realloc(ptr, new_size); } -void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) { - return realloc(ptr, new_size); -} - -void roc_dealloc(void* ptr, unsigned int alignment) { - free(ptr); -} +void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); } void roc_panic(void* ptr, unsigned int alignment) { - char* msg = (char *)ptr; - fprintf(stderr, "Application crashed with message\n\n %s\n\nShutting down\n", msg); - exit(0); + char* msg = (char*)ptr; + fprintf(stderr, + "Application crashed with message\n\n %s\n\nShutting down\n", msg); + exit(0); } +void* roc_memcpy(void* dest, const void* src, size_t n) { + return memcpy(dest, src, n); +} + +void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); } + struct RocStr { - char* bytes; - size_t len; + char* bytes; + size_t len; }; -bool is_small_str(struct RocStr str) { - return ((ssize_t)str.len) < 0; -} +bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; } // Determine the length of the string, taking into // account the small string optimization size_t roc_str_len(struct RocStr str) { - char* bytes = (char*)&str; - char last_byte = bytes[sizeof(str) - 1]; - char last_byte_xored = last_byte ^ 0b10000000; - size_t small_len = (size_t)(last_byte_xored); - size_t big_len = str.len; + char* bytes = (char*)&str; + char last_byte = bytes[sizeof(str) - 1]; + char last_byte_xored = last_byte ^ 0b10000000; + size_t small_len = (size_t)(last_byte_xored); + size_t big_len = str.len; - // Avoid branch misprediction costs by always - // determining both small_len and big_len, - // so this compiles to a cmov instruction. - if (is_small_str(str)) { - return small_len; - } else { - return big_len; - } + // Avoid branch misprediction costs by always + // determining both small_len and big_len, + // so this compiles to a cmov instruction. + if (is_small_str(str)) { + return small_len; + } else { + return big_len; + } } -struct RocCallResult { - size_t flag; - struct RocStr content; -}; - -extern void roc__mainForHost_1_exposed(struct RocCallResult *re); +extern struct RocStr roc__mainForHost_1_exposed(); int main() { - // Make space for the Roc call result - struct RocCallResult call_result; + struct RocStr str = roc__mainForHost_1_exposed(); - // Call Roc to populate call_result - roc__mainForHost_1_exposed(&call_result); + // Determine str_len and the str_bytes pointer, + // taking into account the small string optimization. + size_t str_len = roc_str_len(str); + char* str_bytes; - // Determine str_len and the str_bytes pointer, - // taking into account the small string optimization. - struct RocStr str = call_result.content; - size_t str_len = roc_str_len(str); - char* str_bytes; + if (is_small_str(str)) { + str_bytes = (char*)&str; + } else { + str_bytes = str.bytes; + } - if (is_small_str(str)) { - str_bytes = (char*)&str; - } else { - str_bytes = str.bytes; - } + // Write to stdout + if (write(1, str_bytes, str_len) >= 0) { + // Writing succeeded! + return 0; + } else { + printf("Error writing to stdout: %s\n", strerror(errno)); - // Write to stdout - if (write(1, str_bytes, str_len) >= 0) { - // Writing succeeded! - return 0; - } else { - printf("Error writing to stdout: %s\n", strerror(errno)); - - return 1; - } + return 1; + } } diff --git a/examples/hello-zig/platform/host.zig b/examples/hello-zig/platform/host.zig index 8384c996d3..3c10412eff 100644 --- a/examples/hello-zig/platform/host.zig +++ b/examples/hello-zig/platform/host.zig @@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { _ = alignment; @@ -51,12 +53,18 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(*RocCallResult) void; - -const RocCallResult = extern struct { flag: u64, content: RocStr }; +extern fn roc__mainForHost_1_exposed() RocStr; const Unit = extern struct {}; @@ -64,20 +72,17 @@ pub fn main() u8 { const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = RocStr.empty() }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(&callresult); + var callresult = roc__mainForHost_1_exposed(); // stdout the result - stdout.print("{s}\n", .{callresult.content.asSlice()}) catch unreachable; + stdout.print("{s}\n", .{callresult.asSlice()}) catch unreachable; - callresult.content.deinit(); + callresult.deinit(); // end time var ts2: std.os.timespec = undefined; diff --git a/examples/quicksort/platform/host.zig b/examples/quicksort/platform/host.zig index 38fb29f699..fe8e90e490 100644 --- a/examples/quicksort/platform/host.zig +++ b/examples/quicksort/platform/host.zig @@ -20,12 +20,14 @@ comptime { const mem = std.mem; const Allocator = mem.Allocator; -extern fn roc__mainForHost_1_exposed(RocList, *RocCallResult) void; +extern fn roc__mainForHost_1_exposed(RocList) RocList; const Align = extern struct { a: usize, b: usize }; extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void; extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void; extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void; +extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void; +extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void; const DEBUG: bool = false; @@ -67,13 +69,19 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { std.process.exit(0); } +export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{ + return memcpy(dst, src, size); +} + +export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{ + return memset(dst, value, size); +} + // warning! the array is currently stack-allocated so don't make this too big const NUM_NUMS = 100; const RocList = extern struct { elements: [*]i64, length: usize }; -const RocCallResult = extern struct { flag: u64, content: RocList }; - const Unit = extern struct {}; pub export fn main() u8 { @@ -93,19 +101,16 @@ pub export fn main() u8 { const roc_list = RocList{ .elements = numbers, .length = NUM_NUMS }; - // make space for the result - var callresult = RocCallResult{ .flag = 0, .content = undefined }; - // start time var ts1: std.os.timespec = undefined; std.os.clock_gettime(std.os.CLOCK_REALTIME, &ts1) catch unreachable; // actually call roc to populate the callresult - roc__mainForHost_1_exposed(roc_list, &callresult); + var callresult = roc__mainForHost_1_exposed(roc_list); // stdout the result - const length = std.math.min(20, callresult.content.length); - var result = callresult.content.elements[0..length]; + const length = std.math.min(20, callresult.length); + var result = callresult.elements[0..length]; for (result) |x, i| { if (i == 0) { diff --git a/linker/Cargo.toml b/linker/Cargo.toml new file mode 100644 index 0000000000..843bf6caac --- /dev/null +++ b/linker/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "roc_linker" +version = "0.1.0" +authors = ["The Roc Contributors"] +license = "UPL-1.0" +repository = "https://github.com/rtfeldman/roc" +edition = "2018" +description = "A surgical linker for Roc" + +[lib] +name = "roc_linker" +path = "src/lib.rs" + +[[bin]] +name = "link" +path = "src/main.rs" +test = false +bench = false + +[dependencies] +roc_mono = { path = "../compiler/mono" } +roc_build = { path = "../compiler/build", default-features = false } +roc_collections = { path = "../compiler/collections" } +bumpalo = { version = "3.6", features = ["collections"] } +# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it +clap = { git = "https://github.com/rtfeldman/clap", branch = "master" } +iced-x86 = "1.14" +memmap2 = "0.3" +object = { version = "0.26", features = ["read", "write"] } +serde = { version = "1.0", features = ["derive"] } +bincode = "1.3" +target-lexicon = "0.12.2" +tempfile = "3.1.0" diff --git a/linker/README.md b/linker/README.md new file mode 100644 index 0000000000..0d1e2d77ee --- /dev/null +++ b/linker/README.md @@ -0,0 +1,44 @@ +# The Roc Surgical Linker + +This linker has the goal of being extremely slim lined and fast. +It is focused on the scope of only linking platforms to Roc applications. +This restriction enables ignoring most of linking. + +## General Overview + +This linker is run in 2 phases: preprocessing and surigical linking. + +### Platform Preprocessor + +1. Dynamically link the platform to a dummy Roc application dynamic library +1. Create metadata related to Roc dynamically linked functions + - Symbols that need to be redefined + - Call locations that need to be modified for each symbol + - Locations of special roc functions (roc_alloc, roc_dealloc, builtins, etc) +1. Modify the main executable to no longer be dynamically link + - Delete dependency on dynamic library + - Remove symbols from the dynamic table (maybe add them to the regular table?) + - Delete GOT and PLT entries + - Remove relocations from the dynamic table + - Add extra header information about new text and data section at end of file + +### Surgical Linker + +1. Build off of preprocessed platform +1. Append text and data of application, dealing with app relocations +1. Surgically update all call locations in the platform +1. Surgically update call information in the application (also dealing with other relocations for builtins) + +## TODO (In a lightly prioritized order) + +- Run CLI tests and/or benchmarks with the Roc Linker. +- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig). +- Add Macho support + - Honestly should be almost exactly the same code. + This means we likely need to do a lot of refactoring to minimize the duplicate code. + The fun of almost but not quite the same. +- Add PE support + - As a prereq, we need roc building on Windows (I'm not sure it does currently). + - Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add. +- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl` +- Add a feature to the compiler to make this linker optional. diff --git a/linker/src/lib.rs b/linker/src/lib.rs new file mode 100644 index 0000000000..1a043fbce1 --- /dev/null +++ b/linker/src/lib.rs @@ -0,0 +1,1626 @@ +use bincode::{deserialize_from, serialize_into}; +use clap::{App, AppSettings, Arg, ArgMatches}; +use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind}; +use memmap2::{Mmap, MmapMut}; +use object::write; +use object::{elf, endian}; +use object::{ + Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian, + NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, + SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection, +}; +use roc_build::link::{rebuild_host, LinkType}; +use roc_collections::all::MutMap; +use roc_mono::ir::OptLevel; +use std::cmp::Ordering; +use std::convert::TryFrom; +use std::ffi::CStr; +use std::fs; +use std::io; +use std::io::{BufReader, BufWriter}; +use std::mem; +use std::os::raw::c_char; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; +use std::time::{Duration, SystemTime}; +use target_lexicon::Triple; +use tempfile::Builder; + +mod metadata; + +pub const CMD_PREPROCESS: &str = "preprocess"; +pub const CMD_SURGERY: &str = "surgery"; +pub const FLAG_VERBOSE: &str = "verbose"; +pub const FLAG_TIME: &str = "time"; + +pub const EXEC: &str = "EXEC"; +pub const METADATA: &str = "METADATA"; +pub const SHARED_LIB: &str = "SHARED_LIB"; +pub const APP: &str = "APP"; +pub const OUT: &str = "OUT"; + +const MIN_SECTION_ALIGNMENT: usize = 0x40; + +// TODO: Analyze if this offset is always correct. +const PLT_ADDRESS_OFFSET: u64 = 0x10; + +fn report_timing(label: &str, duration: Duration) { + println!("\t{:9.3} ms {}", duration.as_secs_f64() * 1000.0, label,); +} + +pub fn build_app<'a>() -> App<'a> { + App::new("link") + .about("Preprocesses a platform and surgically links it to an application.") + .setting(AppSettings::SubcommandRequiredElseHelp) + .subcommand( + App::new(CMD_PREPROCESS) + .about("Preprocesses a dynamically linked platform to prepare for linking.") + .arg( + Arg::with_name(EXEC) + .help("The dynamically linked platform executable") + .required(true), + ) + .arg( + Arg::with_name(METADATA) + .help("Where to save the metadata from preprocessing") + .required(true), + ) + .arg( + Arg::with_name(OUT) + .help("The modified version of the dynamically linked platform executable") + .required(true), + ) + .arg( + Arg::with_name(SHARED_LIB) + .help("The name of the shared library used in building the platform") + .default_value("libapp.so"), + ) + .arg( + Arg::with_name(FLAG_VERBOSE) + .long(FLAG_VERBOSE) + .short('v') + .help("Enable verbose printing") + .required(false), + ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .short('t') + .help("Print timing information") + .required(false), + ), + ) + .subcommand( + App::new(CMD_SURGERY) + .about("Links a preprocessed platform with a Roc application.") + .arg( + Arg::with_name(APP) + .help("The Roc application object file waiting to be linked") + .required(true), + ) + .arg( + Arg::with_name(METADATA) + .help("The metadata created by preprocessing the platform") + .required(true), + ) + .arg( + Arg::with_name(OUT) + .help( + "The modified version of the dynamically linked platform. \ + It will be consumed to make linking faster.", + ) + .required(true), + ) + .arg( + Arg::with_name(FLAG_VERBOSE) + .long(FLAG_VERBOSE) + .short('v') + .help("Enable verbose printing") + .required(false), + ) + .arg( + Arg::with_name(FLAG_TIME) + .long(FLAG_TIME) + .short('t') + .help("Print timing information") + .required(false), + ), + ) +} + +pub fn supported(link_type: &LinkType, target: &Triple) -> bool { + link_type == &LinkType::Executable + && target.architecture == target_lexicon::Architecture::X86_64 + && target.operating_system == target_lexicon::OperatingSystem::Linux + && target.binary_format == target_lexicon::BinaryFormat::Elf +} + +pub fn build_and_preprocess_host( + opt_level: OptLevel, + target: &Triple, + host_input_path: &Path, + exposed_to_host: Vec, +) -> io::Result<()> { + let dummy_lib = host_input_path.with_file_name("libapp.so"); + generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?; + rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib)); + let dynhost = host_input_path.with_file_name("dynhost"); + let metadata = host_input_path.with_file_name("metadata"); + let prehost = host_input_path.with_file_name("preprocessedhost"); + if preprocess_impl( + dynhost.to_str().unwrap(), + metadata.to_str().unwrap(), + prehost.to_str().unwrap(), + dummy_lib.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to preprocess host"); + } + Ok(()) +} + +pub fn link_preprocessed_host( + _target: &Triple, + host_input_path: &Path, + roc_app_obj: &Path, + binary_path: &Path, +) -> io::Result<()> { + let metadata = host_input_path.with_file_name("metadata"); + if surgery_impl( + roc_app_obj.to_str().unwrap(), + metadata.to_str().unwrap(), + binary_path.to_str().unwrap(), + false, + false, + )? != 0 + { + panic!("Failed to surgically link host"); + } + Ok(()) +} + +fn generate_dynamic_lib( + _target: &Triple, + exposed_to_host: Vec, + dummy_lib_path: &Path, +) -> io::Result<()> { + let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?; + let dummy_obj_file = dummy_obj_file.path(); + + // TODO deal with other architectures here. + let mut out_object = + write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + + let text_section = out_object.section_id(write::StandardSection::Text); + for sym in exposed_to_host { + // TODO properly generate this list. + for name in &[ + format!("roc__{}_1_exposed", sym), + format!("roc__{}_1_Fx_caller", sym), + format!("roc__{}_1_Fx_size", sym), + format!("roc__{}_1_Fx_result_size", sym), + format!("roc__{}_size", sym), + ] { + out_object.add_symbol(write::Symbol { + name: name.as_bytes().to_vec(), + value: 0, + size: 0, + kind: SymbolKind::Text, + scope: SymbolScope::Dynamic, + weak: false, + section: write::SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } + } + std::fs::write( + &dummy_obj_file, + out_object.write().expect("failed to build output object"), + ) + .expect("failed to write object to file"); + + let output = Command::new("ld") + .args(&[ + "-shared", + "-soname", + dummy_lib_path.file_name().unwrap().to_str().unwrap(), + dummy_obj_file.to_str().unwrap(), + "-o", + dummy_lib_path.to_str().unwrap(), + ]) + .output() + .unwrap(); + + if !output.status.success() { + match std::str::from_utf8(&output.stderr) { + Ok(stderr) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was:\n{}", + stderr + ), + Err(utf8_err) => panic!( + "Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})", + utf8_err + ), + } + } + Ok(()) +} + +pub fn preprocess(matches: &ArgMatches) -> io::Result { + preprocess_impl( + matches.value_of(EXEC).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.value_of(SHARED_LIB).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} +// TODO: Most of this file is a mess of giant functions just to check if things work. +// Clean it all up and refactor nicely. +fn preprocess_impl( + exec_filename: &str, + metadata_filename: &str, + out_filename: &str, + shared_lib_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { + let total_start = SystemTime::now(); + let exec_parsing_start = total_start; + let exec_file = fs::File::open(exec_filename)?; + let exec_mmap = unsafe { Mmap::map(&exec_file)? }; + let exec_data = &*exec_mmap; + let exec_obj = match object::File::parse(exec_data) { + Ok(obj) => obj, + Err(err) => { + println!("Failed to parse executable file: {}", err); + return Ok(-1); + } + }; + let exec_header = load_struct_inplace::>(exec_data, 0); + + let ph_offset = exec_header.e_phoff.get(NativeEndian); + let ph_ent_size = exec_header.e_phentsize.get(NativeEndian); + let ph_num = exec_header.e_phnum.get(NativeEndian); + let sh_offset = exec_header.e_shoff.get(NativeEndian); + let sh_ent_size = exec_header.e_shentsize.get(NativeEndian); + let sh_num = exec_header.e_shnum.get(NativeEndian); + if verbose { + println!(); + println!("PH Offset: {:+x}", ph_offset); + println!("PH Entry Size: {}", ph_ent_size); + println!("PH Entry Count: {}", ph_num); + println!("SH Offset: {:+x}", sh_offset); + println!("SH Entry Size: {}", sh_ent_size); + println!("SH Entry Count: {}", sh_num); + } + + // TODO: Deal with other file formats and architectures. + let format = exec_obj.format(); + if format != BinaryFormat::Elf { + println!("File Format, {:?}, not supported", format); + return Ok(-1); + } + let arch = exec_obj.architecture(); + if arch != Architecture::X86_64 { + println!("Architecture, {:?}, not supported", arch); + return Ok(-1); + } + + let mut md: metadata::Metadata = Default::default(); + + for sym in exec_obj.symbols().filter(|sym| { + sym.is_definition() && sym.name().is_ok() && sym.name().unwrap().starts_with("roc_") + }) { + let name = sym.name().unwrap().to_string(); + // special exceptions for memcpy and memset. + if &name == "roc_memcpy" { + md.roc_symbol_vaddresses + .insert("memcpy".to_string(), sym.address() as u64); + } else if name == "roc_memset" { + md.roc_symbol_vaddresses + .insert("memset".to_string(), sym.address() as u64); + } + md.roc_symbol_vaddresses.insert(name, sym.address() as u64); + } + + if verbose { + println!( + "Found roc symbol definitions: {:+x?}", + md.roc_symbol_vaddresses + ); + } + + let exec_parsing_duration = exec_parsing_start.elapsed().unwrap(); + + // Extract PLT related information for app functions. + let symbol_and_plt_processing_start = SystemTime::now(); + let (plt_address, plt_offset) = match exec_obj.section_by_name(".plt") { + Some(section) => { + let file_offset = match section.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset, + _ => { + println!("Surgical linking does not work with compressed plt section"); + return Ok(-1); + } + }; + (section.address(), file_offset) + } + None => { + println!("Failed to find PLT section. Probably an malformed executable."); + return Ok(-1); + } + }; + if verbose { + println!("PLT Address: {:+x}", plt_address); + println!("PLT File Offset: {:+x}", plt_offset); + } + + // TODO: it looks like we may need to support global data host relocations. + // Rust host look to be using them by default instead of the plt. + // I think this is due to first linking into a static lib and then linking to the c wrapper. + let plt_relocs = (match exec_obj.dynamic_relocations() { + Some(relocs) => relocs, + None => { + println!("Executable never calls any application functions."); + println!("No work to do. Probably an invalid input."); + return Ok(-1); + } + }) + .map(|(_, reloc)| reloc) + .filter(|reloc| reloc.kind() == RelocationKind::Elf(7)); + + let app_syms: Vec = exec_obj + .dynamic_symbols() + .filter(|sym| { + sym.is_undefined() && sym.name().is_ok() && sym.name().unwrap().starts_with("roc_") + }) + .collect(); + for sym in app_syms.iter() { + let name = sym.name().unwrap().to_string(); + md.app_functions.push(name.clone()); + md.surgeries.insert(name.clone(), vec![]); + md.dynamic_symbol_indices.insert(name, sym.index().0 as u64); + } + if verbose { + println!(); + println!("PLT Symbols for App Functions"); + for symbol in app_syms.iter() { + println!("{}: {:+x?}", symbol.index().0, symbol); + } + } + + let mut app_func_addresses: MutMap = MutMap::default(); + for (i, reloc) in plt_relocs.enumerate() { + for symbol in app_syms.iter() { + if reloc.target() == RelocationTarget::Symbol(symbol.index()) { + let func_address = (i as u64 + 1) * PLT_ADDRESS_OFFSET + plt_address; + let func_offset = (i as u64 + 1) * PLT_ADDRESS_OFFSET + plt_offset; + app_func_addresses.insert(func_address, symbol.name().unwrap()); + md.plt_addresses.insert( + symbol.name().unwrap().to_string(), + (func_offset, func_address), + ); + break; + } + } + } + + if verbose { + println!(); + println!("App Function Address Map: {:+x?}", app_func_addresses); + } + let symbol_and_plt_processing_duration = symbol_and_plt_processing_start.elapsed().unwrap(); + + let text_disassembly_start = SystemTime::now(); + let text_sections: Vec
= exec_obj + .sections() + .filter(|sec| { + let name = sec.name(); + name.is_ok() && name.unwrap().starts_with(".text") + }) + .collect(); + if text_sections.is_empty() { + println!("No text sections found. This application has no code."); + return Ok(-1); + } + if verbose { + println!(); + println!("Text Sections"); + for sec in text_sections.iter() { + println!("{:+x?}", sec); + } + } + + if verbose { + println!(); + println!("Analyzing instuctions for branches"); + } + let mut indirect_warning_given = false; + for sec in text_sections { + let (file_offset, compressed) = match sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => (range.offset, false), + Ok(range) => (range.offset, true), + Err(err) => { + println!( + "Issues dealing with section compression for {:+x?}: {}", + sec, err + ); + return Ok(-1); + } + }; + + let data = match sec.uncompressed_data() { + Ok(data) => data, + Err(err) => { + println!("Failed to load text section, {:+x?}: {}", sec, err); + return Ok(-1); + } + }; + let mut decoder = Decoder::with_ip(64, &data, sec.address(), DecoderOptions::NONE); + let mut inst = Instruction::default(); + + while decoder.can_decode() { + decoder.decode_out(&mut inst); + + // Note: This gets really complex fast if we want to support more than basic calls/jumps. + // A lot of them have to load addresses into registers/memory so we would have to discover that value. + // Would probably require some static code analysis and would be impossible in some cases. + // As an alternative we can leave in the calls to the plt, but change the plt to jmp to the static function. + // That way any indirect call will just have the overhead of an extra jump. + match inst.try_op_kind(0) { + // Relative Offsets. + Ok(OpKind::NearBranch16 | OpKind::NearBranch32 | OpKind::NearBranch64) => { + let target = inst.near_branch_target(); + if let Some(func_name) = app_func_addresses.get(&target) { + if compressed { + println!("Surgical linking does not work with compressed text sections: {:+x?}", sec); + return Ok(-1); + } + + if verbose { + println!( + "Found branch from {:+x} to {:+x}({})", + inst.ip(), + target, + func_name + ); + } + + // TODO: Double check these offsets are always correct. + // We may need to do a custom offset based on opcode instead. + let op_kind = inst.op_code().try_op_kind(0).unwrap(); + let op_size: u8 = match op_kind { + OpCodeOperandKind::br16_1 | OpCodeOperandKind::br32_1 => 1, + OpCodeOperandKind::br16_2 => 2, + OpCodeOperandKind::br32_4 | OpCodeOperandKind::br64_4 => 4, + _ => { + println!( + "Ran into an unknown operand kind when analyzing branches: {:?}", + op_kind + ); + return Ok(-1); + } + }; + let offset = inst.next_ip() - op_size as u64 - sec.address() + file_offset; + if verbose { + println!( + "\tNeed to surgically replace {} bytes at file offset {:+x}", + op_size, offset, + ); + println!( + "\tIts current value is {:+x?}", + &exec_data[offset as usize..(offset + op_size as u64) as usize] + ) + } + md.surgeries + .get_mut(*func_name) + .unwrap() + .push(metadata::SurgeryEntry { + file_offset: offset, + virtual_offset: inst.next_ip(), + size: op_size, + }); + } + } + Ok(OpKind::FarBranch16 | OpKind::FarBranch32) => { + println!( + "Found branch type instruction that is not yet support: {:+x?}", + inst + ); + return Ok(-1); + } + Ok(_) => { + if (inst.is_call_far_indirect() + || inst.is_call_near_indirect() + || inst.is_jmp_far_indirect() + || inst.is_jmp_near_indirect()) + && !indirect_warning_given + && verbose + { + indirect_warning_given = true; + println!(); + println!("Cannot analyaze through indirect jmp type instructions"); + println!("Most likely this is not a problem, but it could mean a loss in optimizations"); + println!(); + } + } + Err(err) => { + println!("Failed to decode assembly: {}", err); + return Ok(-1); + } + } + } + } + let text_disassembly_duration = text_disassembly_start.elapsed().unwrap(); + + let scanning_dynamic_deps_start = SystemTime::now(); + + let dyn_sec = match exec_obj.section_by_name(".dynamic") { + Some(sec) => sec, + None => { + println!("There must be a dynamic section in the executable"); + return Ok(-1); + } + }; + let dyn_offset = match dyn_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed dynamic section"); + return Ok(-1); + } + }; + md.dynamic_section_offset = dyn_offset as u64; + + let dynstr_sec = match exec_obj.section_by_name(".dynstr") { + Some(sec) => sec, + None => { + println!("There must be a dynstr section in the executable"); + return Ok(-1); + } + }; + let dynstr_data = match dynstr_sec.uncompressed_data() { + Ok(data) => data, + Err(err) => { + println!("Failed to load dynstr section: {}", err); + return Ok(-1); + } + }; + + let shared_lib_name = Path::new(shared_lib_filename) + .file_name() + .unwrap() + .to_str() + .unwrap(); + + let mut dyn_lib_index = 0; + let mut shared_lib_index = None; + loop { + let dyn_tag = u64::from_le_bytes( + <[u8; 8]>::try_from( + &exec_data[dyn_offset + dyn_lib_index * 16..dyn_offset + dyn_lib_index * 16 + 8], + ) + .unwrap(), + ); + if dyn_tag == 0 { + break; + } else if dyn_tag == 1 { + let dynstr_off = u64::from_le_bytes( + <[u8; 8]>::try_from( + &exec_data + [dyn_offset + dyn_lib_index * 16 + 8..dyn_offset + dyn_lib_index * 16 + 16], + ) + .unwrap(), + ) as usize; + let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8; + let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap(); + if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name { + shared_lib_index = Some(dyn_lib_index); + if verbose { + println!( + "Found shared lib in dynamic table at index: {}", + dyn_lib_index + ); + } + } + } + + dyn_lib_index += 1; + } + let dynamic_lib_count = dyn_lib_index as usize; + + if shared_lib_index.is_none() { + println!("Shared lib not found as a dependency of the executable"); + return Ok(-1); + } + let shared_lib_index = shared_lib_index.unwrap(); + + let scanning_dynamic_deps_duration = scanning_dynamic_deps_start.elapsed().unwrap(); + + let symtab_sec = match exec_obj.section_by_name(".symtab") { + Some(sec) => sec, + None => { + println!("There must be a symtab section in the executable"); + return Ok(-1); + } + }; + let symtab_offset = match symtab_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed symtab section"); + return Ok(-1); + } + }; + md.symbol_table_section_offset = symtab_offset as u64; + md.symbol_table_size = symtab_sec.size(); + + let dynsym_sec = match exec_obj.section_by_name(".dynsym") { + Some(sec) => sec, + None => { + println!("There must be a dynsym section in the executable"); + return Ok(-1); + } + }; + let dynsym_offset = match dynsym_sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => range.offset as usize, + _ => { + println!("Surgical linking does not work with compressed dynsym section"); + return Ok(-1); + } + }; + md.dynamic_symbol_table_section_offset = dynsym_offset as u64; + + let mut got_sections: Vec<(usize, usize)> = vec![]; + for sec in exec_obj + .sections() + .filter(|sec| sec.name().is_ok() && sec.name().unwrap().starts_with(".got")) + { + match sec.compressed_file_range() { + Ok( + range + @ + CompressedFileRange { + format: CompressionFormat::None, + .. + }, + ) => got_sections.push((range.offset as usize, range.uncompressed_size as usize)), + _ => { + println!("Surgical linking does not work with compressed got sections"); + return Ok(-1); + } + } + } + + let platform_gen_start = SystemTime::now(); + + // Copy header and shift everything to enable more program sections. + let added_header_count = 2; + md.added_byte_count = ph_ent_size as u64 * added_header_count; + md.added_byte_count = md.added_byte_count + + (MIN_SECTION_ALIGNMENT as u64 - md.added_byte_count % MIN_SECTION_ALIGNMENT as u64); + let ph_end = ph_offset as usize + ph_num as usize * ph_ent_size as usize; + let physical_shift_start = ph_end as u64; + + md.exec_len = exec_data.len() as u64 + md.added_byte_count; + let out_file = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(out_filename)?; + out_file.set_len(md.exec_len)?; + let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; + + out_mmap[..ph_end].copy_from_slice(&exec_data[..ph_end]); + + let program_headers = load_structs_inplace_mut::>( + &mut out_mmap, + ph_offset as usize, + ph_num as usize, + ); + let mut first_load_found = false; + let mut virtual_shift_start = 0; + for ph in program_headers.iter() { + let p_type = ph.p_type.get(NativeEndian); + if p_type == elf::PT_LOAD && ph.p_offset.get(NativeEndian) == 0 { + first_load_found = true; + md.load_align_constraint = ph.p_align.get(NativeEndian); + virtual_shift_start = physical_shift_start + ph.p_vaddr.get(NativeEndian); + } + } + if !first_load_found { + println!("Executable does not load any data at 0x00000000"); + println!("Probably input the wrong file as the executable"); + return Ok(-1); + } + if verbose { + println!( + "Shifting all data after: {:+x}({:+x})", + physical_shift_start, virtual_shift_start + ); + } + + // Shift all of the program headers. + for ph in program_headers.iter_mut() { + let p_type = ph.p_type.get(NativeEndian); + let p_offset = ph.p_offset.get(NativeEndian); + if (p_type == elf::PT_LOAD && p_offset == 0) || p_type == elf::PT_PHDR { + // Extend length for the first segment and the program header. + ph.p_filesz = endian::U64::new( + LittleEndian, + ph.p_filesz.get(NativeEndian) + md.added_byte_count, + ); + ph.p_memsz = endian::U64::new( + LittleEndian, + ph.p_memsz.get(NativeEndian) + md.added_byte_count, + ); + } else { + // Shift if needed. + if physical_shift_start <= p_offset { + ph.p_offset = endian::U64::new(LittleEndian, p_offset + md.added_byte_count); + } + let p_vaddr = ph.p_vaddr.get(NativeEndian); + if virtual_shift_start <= p_vaddr { + ph.p_vaddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count); + ph.p_paddr = endian::U64::new(LittleEndian, p_vaddr + md.added_byte_count); + } + } + } + + // Get last segment virtual address. + let last_segment_vaddr = program_headers + .iter() + .filter(|ph| ph.p_type.get(NativeEndian) != elf::PT_GNU_STACK) + .map(|ph| ph.p_vaddr.get(NativeEndian) + ph.p_memsz.get(NativeEndian)) + .max() + .unwrap(); + + // Copy the rest of the file shifted as needed. + out_mmap[physical_shift_start as usize + md.added_byte_count as usize..] + .copy_from_slice(&exec_data[physical_shift_start as usize..]); + + // Update all sections for shift for extra program headers. + let section_headers = load_structs_inplace_mut::>( + &mut out_mmap, + sh_offset as usize + md.added_byte_count as usize, + sh_num as usize, + ); + + let mut rel_sections: Vec<(u64, u64)> = vec![]; + let mut rela_sections: Vec<(u64, u64)> = vec![]; + for sh in section_headers.iter_mut() { + let sh_offset = sh.sh_offset.get(NativeEndian); + let sh_addr = sh.sh_addr.get(NativeEndian); + if physical_shift_start <= sh_offset { + sh.sh_offset = endian::U64::new(LittleEndian, sh_offset + md.added_byte_count); + } + if virtual_shift_start <= sh_addr { + sh.sh_addr = endian::U64::new(LittleEndian, sh_addr + md.added_byte_count); + } + + // Record every relocation section. + let sh_type = sh.sh_type.get(NativeEndian); + if sh_type == elf::SHT_REL { + rel_sections.push((sh_offset, sh.sh_size.get(NativeEndian))); + } else if sh_type == elf::SHT_RELA { + rela_sections.push((sh_offset, sh.sh_size.get(NativeEndian))); + } + } + + // Get last section virtual address. + let last_section_vaddr = section_headers + .iter() + .map(|sh| sh.sh_addr.get(NativeEndian) + sh.sh_size.get(NativeEndian)) + .max() + .unwrap(); + + // Calculate end virtual address for new segment. + // TODO: potentially remove md.load_align_constraint here. I think we should be able to cram things together. + md.last_vaddr = + std::cmp::max(last_section_vaddr, last_segment_vaddr) + md.load_align_constraint; + + // Update all relocations for shift for extra program headers. + for (sec_offset, sec_size) in rel_sections { + let relocations = load_structs_inplace_mut::>( + &mut out_mmap, + sec_offset as usize + md.added_byte_count as usize, + sec_size as usize / mem::size_of::>(), + ); + for rel in relocations.iter_mut() { + let r_offset = rel.r_offset.get(NativeEndian); + if virtual_shift_start <= r_offset { + rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count); + } + } + } + for (sec_offset, sec_size) in rela_sections { + let relocations = load_structs_inplace_mut::>( + &mut out_mmap, + sec_offset as usize + md.added_byte_count as usize, + sec_size as usize / mem::size_of::>(), + ); + for rel in relocations.iter_mut() { + let r_offset = rel.r_offset.get(NativeEndian); + if virtual_shift_start <= r_offset { + rel.r_offset = endian::U64::new(LittleEndian, r_offset + md.added_byte_count); + // Deal with potential adjusts to absolute jumps. + // TODO: Verify other relocation types. + if rel.r_type(LittleEndian, false) == elf::R_X86_64_RELATIVE { + let r_addend = rel.r_addend.get(LittleEndian); + rel.r_addend + .set(LittleEndian, r_addend + md.added_byte_count as i64); + } + } + } + } + + // Update dynamic table entries for shift for extra program headers. + let dyn_offset = md.dynamic_section_offset + md.added_byte_count; + + let dyns = load_structs_inplace_mut::>( + &mut out_mmap, + dyn_offset as usize, + dynamic_lib_count, + ); + for mut d in dyns { + match d.d_tag.get(NativeEndian) as u32 { + // I believe this is the list of symbols that need to be update if addresses change. + // I am less sure about the symbols from GNU_HASH down. + elf::DT_INIT + | elf::DT_FINI + | elf::DT_PLTGOT + | elf::DT_HASH + | elf::DT_STRTAB + | elf::DT_SYMTAB + | elf::DT_RELA + | elf::DT_REL + | elf::DT_DEBUG + | elf::DT_JMPREL + | elf::DT_INIT_ARRAY + | elf::DT_FINI_ARRAY + | elf::DT_PREINIT_ARRAY + | elf::DT_SYMTAB_SHNDX + | elf::DT_GNU_HASH + | elf::DT_TLSDESC_PLT + | elf::DT_TLSDESC_GOT + | elf::DT_GNU_CONFLICT + | elf::DT_GNU_LIBLIST + | elf::DT_CONFIG + | elf::DT_DEPAUDIT + | elf::DT_AUDIT + | elf::DT_PLTPAD + | elf::DT_MOVETAB + | elf::DT_SYMINFO + | elf::DT_VERSYM + | elf::DT_VERDEF + | elf::DT_VERNEED => { + let d_addr = d.d_val.get(NativeEndian); + if virtual_shift_start <= d_addr { + d.d_val = endian::U64::new(LittleEndian, d_addr + md.added_byte_count); + } + } + _ => {} + } + } + + // Update symbol table entries for shift for extra program headers. + let symtab_offset = md.symbol_table_section_offset + md.added_byte_count; + let symtab_size = md.symbol_table_size as usize; + + let symbols = load_structs_inplace_mut::>( + &mut out_mmap, + symtab_offset as usize, + symtab_size / mem::size_of::>(), + ); + + for sym in symbols { + let addr = sym.st_value.get(NativeEndian); + if virtual_shift_start <= addr { + sym.st_value = endian::U64::new(LittleEndian, addr + md.added_byte_count); + } + } + + // Update all data in the global offset table. + for (offset, size) in got_sections { + let global_offsets = load_structs_inplace_mut::>( + &mut out_mmap, + offset as usize + md.added_byte_count as usize, + size / mem::size_of::>(), + ); + for go in global_offsets.iter_mut() { + let go_addr = go.get(NativeEndian); + if physical_shift_start <= go_addr { + go.set(LittleEndian, go_addr + md.added_byte_count); + } + } + } + + // TODO: look into shifting all of the debug info and eh_frames. + + // Delete shared library from the dynamic table. + let out_ptr = out_mmap.as_mut_ptr(); + unsafe { + std::ptr::copy( + out_ptr.add(dyn_offset as usize + 16 * (shared_lib_index + 1)), + out_ptr.add(dyn_offset as usize + 16 * shared_lib_index), + 16 * (dynamic_lib_count - shared_lib_index), + ); + } + + // Update main elf header for extra data. + let mut file_header = + load_struct_inplace_mut::>(&mut out_mmap, 0); + file_header.e_shoff = endian::U64::new( + LittleEndian, + file_header.e_shoff.get(NativeEndian) + md.added_byte_count, + ); + let e_entry = file_header.e_entry.get(NativeEndian); + if virtual_shift_start <= e_entry { + file_header.e_entry = endian::U64::new(LittleEndian, e_entry + md.added_byte_count); + } + file_header.e_phnum = endian::U16::new(LittleEndian, ph_num + added_header_count as u16); + + let platform_gen_duration = platform_gen_start.elapsed().unwrap(); + + if verbose { + println!(); + println!("{:+x?}", md); + } + + let saving_metadata_start = SystemTime::now(); + // This block ensure that the metadata is fully written and timed before continuing. + { + let output = fs::File::create(metadata_filename)?; + let output = BufWriter::new(output); + if let Err(err) = serialize_into(output, &md) { + println!("Failed to serialize metadata: {}", err); + return Ok(-1); + }; + } + let saving_metadata_duration = saving_metadata_start.elapsed().unwrap(); + + let flushing_data_start = SystemTime::now(); + out_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(out_mmap); + drop(out_file); + let flushing_data_duration = flushing_data_start.elapsed().unwrap(); + + let total_duration = total_start.elapsed().unwrap(); + + if verbose || time { + println!(); + println!("Timings"); + report_timing("Executable Parsing", exec_parsing_duration); + report_timing( + "Symbol and PLT Processing", + symbol_and_plt_processing_duration, + ); + report_timing("Text Disassembly", text_disassembly_duration); + report_timing("Scanning Dynamic Deps", scanning_dynamic_deps_duration); + report_timing("Generate Modified Platform", platform_gen_duration); + report_timing("Saving Metadata", saving_metadata_duration); + report_timing("Flushing Data to Disk", flushing_data_duration); + report_timing( + "Other", + total_duration + - exec_parsing_duration + - symbol_and_plt_processing_duration + - text_disassembly_duration + - scanning_dynamic_deps_duration + - platform_gen_duration + - saving_metadata_duration + - flushing_data_duration, + ); + report_timing("Total", total_duration); + } + + Ok(0) +} + +pub fn surgery(matches: &ArgMatches) -> io::Result { + surgery_impl( + matches.value_of(APP).unwrap(), + matches.value_of(METADATA).unwrap(), + matches.value_of(OUT).unwrap(), + matches.is_present(FLAG_VERBOSE), + matches.is_present(FLAG_TIME), + ) +} + +fn surgery_impl( + app_filename: &str, + metadata_filename: &str, + out_filename: &str, + verbose: bool, + time: bool, +) -> io::Result { + let total_start = SystemTime::now(); + let loading_metadata_start = total_start; + let input = fs::File::open(metadata_filename)?; + let input = BufReader::new(input); + let md: metadata::Metadata = match deserialize_from(input) { + Ok(data) => data, + Err(err) => { + println!("Failed to deserialize metadata: {}", err); + return Ok(-1); + } + }; + let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); + + let app_parsing_start = SystemTime::now(); + let app_file = fs::File::open(app_filename)?; + let app_mmap = unsafe { Mmap::map(&app_file)? }; + let app_data = &*app_mmap; + let app_obj = match object::File::parse(app_data) { + Ok(obj) => obj, + Err(err) => { + println!("Failed to parse application file: {}", err); + return Ok(-1); + } + }; + let app_parsing_duration = app_parsing_start.elapsed().unwrap(); + + let exec_parsing_start = SystemTime::now(); + let exec_file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(out_filename)?; + + let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; + exec_file.set_len(max_out_len)?; + + let mut exec_mmap = unsafe { MmapMut::map_mut(&exec_file)? }; + let elf64 = exec_mmap[4] == 2; + let litte_endian = exec_mmap[5] == 1; + if !elf64 || !litte_endian { + println!("Only 64bit little endian elf currently supported for surgery"); + return Ok(-1); + } + let exec_header = load_struct_inplace::>(&exec_mmap, 0); + + let ph_offset = exec_header.e_phoff.get(NativeEndian); + let ph_ent_size = exec_header.e_phentsize.get(NativeEndian); + let ph_num = exec_header.e_phnum.get(NativeEndian); + let sh_offset = exec_header.e_shoff.get(NativeEndian); + let sh_ent_size = exec_header.e_shentsize.get(NativeEndian); + let sh_num = exec_header.e_shnum.get(NativeEndian); + if verbose { + println!(); + println!("Is Elf64: {}", elf64); + println!("Is Little Endian: {}", litte_endian); + println!("PH Offset: {:+x}", ph_offset); + println!("PH Entry Size: {}", ph_ent_size); + println!("PH Entry Count: {}", ph_num); + println!("SH Offset: {:+x}", sh_offset); + println!("SH Entry Size: {}", sh_ent_size); + println!("SH Entry Count: {}", sh_num); + } + let exec_parsing_duration = exec_parsing_start.elapsed().unwrap(); + + let out_gen_start = SystemTime::now(); + // Backup section header table. + let sh_size = sh_ent_size as usize * sh_num as usize; + let mut sh_tab = vec![]; + sh_tab.extend_from_slice(&exec_mmap[sh_offset as usize..sh_offset as usize + sh_size]); + + let mut offset = sh_offset as usize; + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + + let new_rodata_section_offset = offset; + + // Align physical and virtual address of new segment. + let mut virt_offset = align_to_offset_by_constraint( + md.last_vaddr as usize, + offset, + md.load_align_constraint as usize, + ); + let new_rodata_section_vaddr = virt_offset; + if verbose { + println!(); + println!( + "New Virtual Rodata Section Address: {:+x?}", + new_rodata_section_vaddr + ); + } + + // First decide on sections locations and then recode every exact symbol locations. + + // Copy sections and resolve their symbols/relocations. + let symbols = app_obj.symbols().collect::>(); + let mut section_offset_map: MutMap = MutMap::default(); + let mut symbol_vaddr_map: MutMap = MutMap::default(); + let mut app_func_vaddr_map: MutMap = MutMap::default(); + let mut app_func_size_map: MutMap = MutMap::default(); + + // TODO: Does Roc ever create a data section? I think no cause it would mess up fully functional guarantees. + // If not we never need to think about it, but we should double check. + let rodata_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".rodata")) + .collect(); + + // bss section is like rodata section, but it has zero file size and non-zero virtual size. + let bss_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".bss")) + .collect(); + + let text_sections: Vec
= app_obj + .sections() + .filter(|sec| sec.name().unwrap_or_default().starts_with(".text")) + .collect(); + if text_sections.is_empty() { + println!("No text sections found. This application has no code."); + return Ok(-1); + } + + // Calculate addresses and load symbols. + // Note, it is important the bss sections come after the rodata sections. + for sec in rodata_sections + .iter() + .chain(bss_sections.iter()) + .chain(text_sections.iter()) + { + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + virt_offset = + align_to_offset_by_constraint(virt_offset, offset, md.load_align_constraint as usize); + if verbose { + println!( + "Section, {}, is being put at offset: {:+x}(virt: {:+x})", + sec.name().unwrap(), + offset, + virt_offset + ) + } + section_offset_map.insert(sec.index(), (offset, virt_offset)); + for sym in symbols.iter() { + if sym.section() == SymbolSection::Section(sec.index()) { + let name = sym.name().unwrap_or_default().to_string(); + if !md.roc_symbol_vaddresses.contains_key(&name) { + symbol_vaddr_map.insert(sym.index(), virt_offset + sym.address() as usize); + } + if md.app_functions.contains(&name) { + app_func_vaddr_map.insert(name.clone(), virt_offset + sym.address() as usize); + app_func_size_map.insert(name, sym.size()); + } + } + } + let section_size = match sec.file_range() { + Some((_, size)) => size, + None => 0, + }; + if sec.name().unwrap_or_default().starts_with(".bss") { + // bss sections only modify the virtual size. + virt_offset += sec.size() as usize; + } else if section_size != sec.size() { + println!( + "We do not deal with non bss sections that have different on disk and in memory sizes" + ); + return Ok(-1); + } else { + offset += section_size as usize; + virt_offset += sec.size() as usize; + } + } + if verbose { + println!("Data Relocation Offsets: {:+x?}", symbol_vaddr_map); + println!("Found App Function Symbols: {:+x?}", app_func_vaddr_map); + } + + let (new_text_section_offset, new_text_section_vaddr) = text_sections + .iter() + .map(|sec| section_offset_map.get(&sec.index()).unwrap()) + .min() + .unwrap(); + let (new_text_section_offset, new_text_section_vaddr) = + (*new_text_section_offset, *new_text_section_vaddr); + + // Move data and deal with relocations. + for sec in rodata_sections + .iter() + .chain(bss_sections.iter()) + .chain(text_sections.iter()) + { + let data = match sec.data() { + Ok(data) => data, + Err(err) => { + println!( + "Failed to load data for section, {:+x?}: {}", + sec.name().unwrap(), + err + ); + return Ok(-1); + } + }; + let (section_offset, section_virtual_offset) = + section_offset_map.get(&sec.index()).unwrap(); + let (section_offset, section_virtual_offset) = (*section_offset, *section_virtual_offset); + exec_mmap[section_offset..section_offset + data.len()].copy_from_slice(data); + // Deal with definitions and relocations for this section. + if verbose { + println!(); + println!( + "Processing Relocations for Section: {:+x?} @ {:+x} (virt: {:+x})", + sec, section_offset, section_virtual_offset + ); + } + for rel in sec.relocations() { + if verbose { + println!("\tFound Relocation: {:+x?}", rel); + } + match rel.1.target() { + RelocationTarget::Symbol(index) => { + let target_offset = if let Some(target_offset) = symbol_vaddr_map.get(&index) { + if verbose { + println!( + "\t\tRelocation targets symbol in app at: {:+x}", + target_offset + ); + } + Some(*target_offset as i64) + } else { + app_obj + .symbol_by_index(index) + .and_then(|sym| sym.name()) + .ok() + .and_then(|name| { + md.roc_symbol_vaddresses.get(name).map(|address| { + let vaddr = (*address + md.added_byte_count) as i64; + if verbose { + println!( + "\t\tRelocation targets symbol in host: {} @ {:+x}", + name, vaddr + ); + } + vaddr + }) + }) + }; + + if let Some(target_offset) = target_offset { + let virt_base = section_virtual_offset as usize + rel.0 as usize; + let base = section_offset as usize + rel.0 as usize; + let target: i64 = match rel.1.kind() { + RelocationKind::Relative | RelocationKind::PltRelative => { + target_offset - virt_base as i64 + rel.1.addend() + } + x => { + println!("Relocation Kind not yet support: {:?}", x); + return Ok(-1); + } + }; + if verbose { + println!( + "\t\tRelocation base location: {:+x} (virt: {:+x})", + base, virt_base + ); + println!("\t\tFinal relocation target offset: {:+x}", target); + } + match rel.1.size() { + 32 => { + let data = (target as i32).to_le_bytes(); + exec_mmap[base..base + 4].copy_from_slice(&data); + } + 64 => { + let data = target.to_le_bytes(); + exec_mmap[base..base + 8].copy_from_slice(&data); + } + x => { + println!("Relocation size not yet supported: {}", x); + return Ok(-1); + } + } + } else if matches!(app_obj.symbol_by_index(index), Ok(sym) if ["__divti3", "__udivti3"].contains(&sym.name().unwrap_or_default())) + { + // Explicitly ignore some symbols that are currently always linked. + continue; + } else { + println!( + "Undefined Symbol in relocation, {:+x?}: {:+x?}", + rel, + app_obj.symbol_by_index(index) + ); + return Ok(-1); + } + } + + _ => { + println!("Relocation target not yet support: {:+x?}", rel); + return Ok(-1); + } + } + } + } + + offset = align_by_constraint(offset, MIN_SECTION_ALIGNMENT); + let new_sh_offset = offset; + exec_mmap[offset..offset + sh_size].copy_from_slice(&sh_tab); + offset += sh_size; + + // Flush app only data to speed up write to disk. + exec_mmap.flush_async_range( + new_rodata_section_offset, + offset - new_rodata_section_offset, + )?; + + // TODO: look into merging symbol tables, debug info, and eh frames to enable better debugger experience. + + // Add 2 new sections and segments. + let new_section_count = 2; + offset += new_section_count * sh_ent_size as usize; + let section_headers = load_structs_inplace_mut::>( + &mut exec_mmap, + new_sh_offset as usize, + sh_num as usize + new_section_count, + ); + + let new_rodata_section_size = new_text_section_offset as u64 - new_rodata_section_offset as u64; + let new_rodata_section_virtual_size = + new_text_section_vaddr as u64 - new_rodata_section_vaddr as u64; + let new_text_section_vaddr = new_rodata_section_vaddr as u64 + new_rodata_section_size as u64; + let new_text_section_size = new_sh_offset as u64 - new_text_section_offset as u64; + + let new_rodata_section = &mut section_headers[section_headers.len() - 2]; + new_rodata_section.sh_name = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_type = endian::U32::new(LittleEndian, elf::SHT_PROGBITS); + new_rodata_section.sh_flags = endian::U64::new(LittleEndian, (elf::SHF_ALLOC) as u64); + new_rodata_section.sh_addr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_section.sh_offset = endian::U64::new(LittleEndian, new_rodata_section_offset as u64); + new_rodata_section.sh_size = endian::U64::new(LittleEndian, new_rodata_section_size); + new_rodata_section.sh_link = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_info = endian::U32::new(LittleEndian, 0); + new_rodata_section.sh_addralign = endian::U64::new(LittleEndian, 16); + new_rodata_section.sh_entsize = endian::U64::new(LittleEndian, 0); + + let new_text_section_index = section_headers.len() - 1; + let new_text_section = &mut section_headers[new_text_section_index]; + new_text_section.sh_name = endian::U32::new(LittleEndian, 0); + new_text_section.sh_type = endian::U32::new(LittleEndian, elf::SHT_PROGBITS); + new_text_section.sh_flags = + endian::U64::new(LittleEndian, (elf::SHF_ALLOC | elf::SHF_EXECINSTR) as u64); + new_text_section.sh_addr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_section.sh_offset = endian::U64::new(LittleEndian, new_text_section_offset as u64); + new_text_section.sh_size = endian::U64::new(LittleEndian, new_text_section_size); + new_text_section.sh_link = endian::U32::new(LittleEndian, 0); + new_text_section.sh_info = endian::U32::new(LittleEndian, 0); + new_text_section.sh_addralign = endian::U64::new(LittleEndian, 16); + new_text_section.sh_entsize = endian::U64::new(LittleEndian, 0); + + // Reload and update file header and size. + let file_header = load_struct_inplace_mut::>(&mut exec_mmap, 0); + file_header.e_shoff = endian::U64::new(LittleEndian, new_sh_offset as u64); + file_header.e_shnum = endian::U16::new(LittleEndian, sh_num + new_section_count as u16); + + // Add 2 new segments that match the new sections. + let program_headers = load_structs_inplace_mut::>( + &mut exec_mmap, + ph_offset as usize, + ph_num as usize, + ); + let new_rodata_segment = &mut program_headers[program_headers.len() - 2]; + new_rodata_segment.p_type = endian::U32::new(LittleEndian, elf::PT_LOAD); + new_rodata_segment.p_flags = endian::U32::new(LittleEndian, elf::PF_R); + new_rodata_segment.p_offset = endian::U64::new(LittleEndian, new_rodata_section_offset as u64); + new_rodata_segment.p_vaddr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_segment.p_paddr = endian::U64::new(LittleEndian, new_rodata_section_vaddr as u64); + new_rodata_segment.p_filesz = endian::U64::new(LittleEndian, new_rodata_section_size); + new_rodata_segment.p_memsz = endian::U64::new(LittleEndian, new_rodata_section_virtual_size); + new_rodata_segment.p_align = endian::U64::new(LittleEndian, md.load_align_constraint); + + let new_text_segment = &mut program_headers[program_headers.len() - 1]; + new_text_segment.p_type = endian::U32::new(LittleEndian, elf::PT_LOAD); + new_text_segment.p_flags = endian::U32::new(LittleEndian, elf::PF_R | elf::PF_X); + new_text_segment.p_offset = endian::U64::new(LittleEndian, new_text_section_offset as u64); + new_text_segment.p_vaddr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_segment.p_paddr = endian::U64::new(LittleEndian, new_text_section_vaddr); + new_text_segment.p_filesz = endian::U64::new(LittleEndian, new_text_section_size); + new_text_segment.p_memsz = endian::U64::new(LittleEndian, new_text_section_size); + new_text_segment.p_align = endian::U64::new(LittleEndian, md.load_align_constraint); + + // Update calls from platform and dynamic symbols. + let dynsym_offset = md.dynamic_symbol_table_section_offset + md.added_byte_count; + + for func_name in md.app_functions { + let virt_offset = match app_func_vaddr_map.get(&func_name) { + Some(offset) => *offset as u64, + None => { + println!("Function, {}, was not defined by the app", &func_name); + return Ok(-1); + } + }; + if verbose { + println!( + "Updating calls to {} to the address: {:+x}", + &func_name, virt_offset + ); + } + + for s in md.surgeries.get(&func_name).unwrap_or(&vec![]) { + if verbose { + println!("\tPerforming surgery: {:+x?}", s); + } + match s.size { + 4 => { + let target = (virt_offset as i64 + - (s.virtual_offset + md.added_byte_count) as i64) + as i32; + if verbose { + println!("\tTarget Jump: {:+x}", target); + } + let data = target.to_le_bytes(); + exec_mmap[(s.file_offset + md.added_byte_count) as usize + ..(s.file_offset + md.added_byte_count) as usize + 4] + .copy_from_slice(&data); + } + x => { + println!("Surgery size not yet supported: {}", x); + return Ok(-1); + } + } + } + + // Replace plt call code with just a jump. + // This is a backup incase we missed a call to the plt. + if let Some((plt_off, plt_vaddr)) = md.plt_addresses.get(&func_name) { + let plt_off = (*plt_off + md.added_byte_count) as usize; + let plt_vaddr = *plt_vaddr + md.added_byte_count; + let jmp_inst_len = 5; + let target = (virt_offset as i64 - (plt_vaddr as i64 + jmp_inst_len as i64)) as i32; + if verbose { + println!("\tPLT: {:+x}, {:+x}", plt_off, plt_vaddr); + println!("\tTarget Jump: {:+x}", target); + } + let data = target.to_le_bytes(); + exec_mmap[plt_off] = 0xE9; + exec_mmap[plt_off + 1..plt_off + jmp_inst_len].copy_from_slice(&data); + for i in jmp_inst_len..PLT_ADDRESS_OFFSET as usize { + exec_mmap[plt_off + i] = 0x90; + } + } + + if let Some(i) = md.dynamic_symbol_indices.get(&func_name) { + let sym = load_struct_inplace_mut::>( + &mut exec_mmap, + dynsym_offset as usize + *i as usize * mem::size_of::>(), + ); + sym.st_shndx = endian::U16::new(LittleEndian, new_text_section_index as u16); + sym.st_value = endian::U64::new(LittleEndian, virt_offset as u64); + sym.st_size = endian::U64::new( + LittleEndian, + match app_func_size_map.get(&func_name) { + Some(size) => *size, + None => { + println!("Size missing for: {}", &func_name); + return Ok(-1); + } + }, + ); + } + } + + let out_gen_duration = out_gen_start.elapsed().unwrap(); + + let flushing_data_start = SystemTime::now(); + exec_mmap.flush()?; + // Also drop files to to ensure data is fully written here. + drop(exec_mmap); + exec_file.set_len(offset as u64 + 1)?; + drop(exec_file); + let flushing_data_duration = flushing_data_start.elapsed().unwrap(); + + // Make sure the final executable has permision to execute. + let mut perms = fs::metadata(out_filename)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(out_filename, perms)?; + + let total_duration = total_start.elapsed().unwrap(); + + if verbose || time { + println!(); + println!("Timings"); + report_timing("Loading Metadata", loading_metadata_duration); + report_timing("Executable Parsing", exec_parsing_duration); + report_timing("Application Parsing", app_parsing_duration); + report_timing("Output Generation", out_gen_duration); + report_timing("Flushing Data to Disk", flushing_data_duration); + report_timing( + "Other", + total_duration + - loading_metadata_duration + - exec_parsing_duration + - app_parsing_duration + - out_gen_duration + - flushing_data_duration, + ); + report_timing("Total", total_duration); + } + Ok(0) +} + +fn align_by_constraint(offset: usize, constraint: usize) -> usize { + if offset % constraint == 0 { + offset + } else { + offset + constraint - (offset % constraint) + } +} + +fn align_to_offset_by_constraint( + current_offset: usize, + target_offset: usize, + constraint: usize, +) -> usize { + let target_remainder = target_offset % constraint; + let current_remainder = current_offset % constraint; + match target_remainder.cmp(¤t_remainder) { + Ordering::Greater => current_offset + (target_remainder - current_remainder), + Ordering::Less => current_offset + ((target_remainder + constraint) - current_remainder), + Ordering::Equal => current_offset, + } +} + +fn load_struct_inplace(bytes: &[u8], offset: usize) -> &T { + &load_structs_inplace(bytes, offset, 1)[0] +} + +fn load_struct_inplace_mut(bytes: &mut [u8], offset: usize) -> &mut T { + &mut load_structs_inplace_mut(bytes, offset, 1)[0] +} + +fn load_structs_inplace(bytes: &[u8], offset: usize, count: usize) -> &[T] { + let (head, body, tail) = + unsafe { bytes[offset..offset + count * mem::size_of::()].align_to::() }; + assert!(head.is_empty(), "Data was not aligned"); + assert_eq!(count, body.len(), "Failed to load all structs"); + assert!(tail.is_empty(), "End of data was not aligned"); + body +} + +fn load_structs_inplace_mut(bytes: &mut [u8], offset: usize, count: usize) -> &mut [T] { + let (head, body, tail) = + unsafe { bytes[offset..offset + count * mem::size_of::()].align_to_mut::() }; + assert!(head.is_empty(), "Data was not aligned"); + assert_eq!(count, body.len(), "Failed to load all structs"); + assert!(tail.is_empty(), "End of data was not aligned"); + body +} diff --git a/linker/src/main.rs b/linker/src/main.rs new file mode 100644 index 0000000000..6c1f341f9d --- /dev/null +++ b/linker/src/main.rs @@ -0,0 +1,20 @@ +use roc_linker::{build_app, preprocess, surgery, CMD_PREPROCESS, CMD_SURGERY}; +use std::io; + +fn main() -> io::Result<()> { + let matches = build_app().get_matches(); + + let exit_code = match matches.subcommand_name() { + None => Ok::(-1), + Some(CMD_PREPROCESS) => { + let sub_matches = matches.subcommand_matches(CMD_PREPROCESS).unwrap(); + preprocess(sub_matches) + } + Some(CMD_SURGERY) => { + let sub_matches = matches.subcommand_matches(CMD_SURGERY).unwrap(); + surgery(sub_matches) + } + _ => unreachable!(), + }?; + std::process::exit(exit_code); +} diff --git a/linker/src/metadata.rs b/linker/src/metadata.rs new file mode 100644 index 0000000000..16f06232cd --- /dev/null +++ b/linker/src/metadata.rs @@ -0,0 +1,31 @@ +use roc_collections::all::MutMap; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +pub struct SurgeryEntry { + pub file_offset: u64, + pub virtual_offset: u64, + pub size: u8, +} + +// TODO: Reanalyze each piece of data in this struct. +// I think a number of them can be combined to reduce string duplication. +// Also I think a few of them aren't need. +// For example, I think preprocessing can deal with all shifting and remove the need for added_byte_count. +#[derive(Default, Serialize, Deserialize, PartialEq, Debug)] +pub struct Metadata { + pub app_functions: Vec, + // offset followed by address. + pub plt_addresses: MutMap, + pub surgeries: MutMap>, + pub dynamic_symbol_indices: MutMap, + pub roc_symbol_vaddresses: MutMap, + pub exec_len: u64, + pub load_align_constraint: u64, + pub added_byte_count: u64, + pub last_vaddr: u64, + pub dynamic_section_offset: u64, + pub dynamic_symbol_table_section_offset: u64, + pub symbol_table_section_offset: u64, + pub symbol_table_size: u64, +} diff --git a/linker/tests/fib/.gitignore b/linker/tests/fib/.gitignore new file mode 100644 index 0000000000..a3c0d77f6d --- /dev/null +++ b/linker/tests/fib/.gitignore @@ -0,0 +1,11 @@ +fib + +zig-cache +zig-out + +*.o + +dynhost +preprocessedhost +metadata +libapp.so \ No newline at end of file diff --git a/linker/tests/fib/Main.roc b/linker/tests/fib/Main.roc new file mode 100644 index 0000000000..646fdbea75 --- /dev/null +++ b/linker/tests/fib/Main.roc @@ -0,0 +1,15 @@ +app "fib" + packages { base: "platform" } + imports [] + provides [ main ] to base + +main : U64 -> U64 +main = \index -> + fibHelp index 0 1 + +fibHelp : U64, U64, U64 -> U64 +fibHelp = \index, parent, grandparent -> + if index == 0 then + parent + else + fibHelp (index - 1) grandparent (parent + grandparent) \ No newline at end of file diff --git a/linker/tests/fib/README.md b/linker/tests/fib/README.md new file mode 100644 index 0000000000..0f1af10077 --- /dev/null +++ b/linker/tests/fib/README.md @@ -0,0 +1,48 @@ +# Hello, World! + +To run, `cd` into this directory and run: + +```bash +$ cargo run Hello.roc +``` + +To run in release mode instead, do: + +```bash +$ cargo run --release Hello.roc +``` + +## Troubleshooting + +If you encounter `cannot find -lc++`, run the following for ubuntu `sudo apt install libc++-dev`. + +## Design Notes + +This demonstrates the basic design of hosts: Roc code gets compiled into a pure +function (in this case, a thunk that always returns `"Hello, World!"`) and +then the host calls that function. Fundamentally, that's the whole idea! The host +might not even have a `main` - it could be a library, a plugin, anything. +Everything else is built on this basic "hosts calling linked pure functions" design. + +For example, things get more interesting when the compiled Roc function returns +a `Task` - that is, a tagged union data structure containing function pointers +to callback closures. This lets the Roc pure function describe arbitrary +chainable effects, which the host can interpret to perform I/O as requested by +the Roc program. (The tagged union `Task` would have a variant for each supported +I/O operation.) + +In this trivial example, it's very easy to line up the API between the host and +the Roc program. In a more involved host, this would be much trickier - especially +if the API were changing frequently during development. + +The idea there is to have a first-class concept of "glue code" which host authors +can write (it would be plain Roc code, but with some extra keywords that aren't +available in normal modules - kinda like `port module` in Elm), and which +describe both the Roc-host/C boundary as well as the Roc-host/Roc-app boundary. +Roc application authors only care about the Roc-host/Roc-app portion, and the +host author only cares about the Roc-host/C boundary when implementing the host. + +Using this glue code, the Roc compiler can generate C header files describing the +boundary. This not only gets us host compatibility with C compilers, but also +Rust FFI for free, because [`rust-bindgen`](https://github.com/rust-lang/rust-bindgen) +generates correct Rust FFI bindings from C headers. diff --git a/linker/tests/fib/platform/Package-Config.roc b/linker/tests/fib/platform/Package-Config.roc new file mode 100644 index 0000000000..d93cd7c258 --- /dev/null +++ b/linker/tests/fib/platform/Package-Config.roc @@ -0,0 +1,10 @@ +platform tests/fib + requires {}{ main : U64 -> U64 } + exposes [] + packages {} + imports [] + provides [ mainForHost ] + effects fx.Effect {} + +mainForHost : U64 -> U64 +mainForHost = \arg -> main arg # workaround for https://github.com/rtfeldman/roc/issues/1622 \ No newline at end of file diff --git a/linker/tests/fib/platform/app.zig b/linker/tests/fib/platform/app.zig new file mode 100644 index 0000000000..105908633f --- /dev/null +++ b/linker/tests/fib/platform/app.zig @@ -0,0 +1 @@ +export fn roc__mainForHost_1_exposed(_i: i64, _result: *u64) void {} diff --git a/linker/tests/fib/platform/build.zig b/linker/tests/fib/platform/build.zig new file mode 100644 index 0000000000..deb36d6c78 --- /dev/null +++ b/linker/tests/fib/platform/build.zig @@ -0,0 +1,33 @@ +const Builder = @import("std").build.Builder; + +pub fn build(b: *Builder) void { + // Standard target options allows the person running `zig build` to choose + // what target to build for. Here we do not override the defaults, which + // means any target is allowed, and the default is native. Other options + // for restricting supported target set are available. + const target = b.standardTargetOptions(.{}); + + // Standard release options allow the person running `zig build` to select + // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. + const mode = b.standardReleaseOptions(); + + const app = b.addSharedLibrary("app", "app.zig", .unversioned); + app.setTarget(target); + app.setBuildMode(mode); + app.install(); + + const exe = b.addExecutable("dynhost", "host.zig"); + exe.pie = true; + exe.strip = true; + exe.setTarget(target); + exe.setBuildMode(mode); + exe.linkLibrary(app); + exe.linkLibC(); + exe.install(); + + const run_cmd = exe.run(); + run_cmd.step.dependOn(b.getInstallStep()); + + const run_step = b.step("run", "Run the app"); + run_step.dependOn(&run_cmd.step); +} diff --git a/linker/tests/fib/platform/host.zig b/linker/tests/fib/platform/host.zig new file mode 100644 index 0000000000..c439c889c7 --- /dev/null +++ b/linker/tests/fib/platform/host.zig @@ -0,0 +1,73 @@ +const std = @import("std"); +const testing = std.testing; +const expectEqual = testing.expectEqual; +const expect = testing.expect; + +comptime { + // This is a workaround for https://github.com/ziglang/zig/issues/8218 + // which is only necessary on macOS. + // + // Once that issue is fixed, we can undo the changes in + // 177cf12e0555147faa4d436e52fc15175c2c4ff0 and go back to passing + // -fcompiler-rt in link.rs instead of doing this. Note that this + // workaround is present in many host.zig files, so make sure to undo + // it everywhere! + if (std.builtin.os.tag == .macos) { + _ = @import("compiler_rt"); + } +} + +extern fn malloc(size: usize) callconv(.C) ?*c_void; +extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void; +extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void; + +export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void { + return malloc(size); +} + +export fn roc_realloc(c_ptr: *c_void, old_size: usize, new_size: usize, alignment: u32) callconv(.C) ?*c_void { + return realloc(@alignCast(16, @ptrCast([*]u8, c_ptr)), new_size); +} + +export fn roc_dealloc(c_ptr: *c_void, alignment: u32) callconv(.C) void { + free(@alignCast(16, @ptrCast([*]u8, c_ptr))); +} + +export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void { + const stderr = std.io.getStdErr().writer(); + const msg = @ptrCast([*:0]const u8, c_ptr); + stderr.print("Application crashed with message\n\n {s}\n\nShutting down\n", .{msg}) catch unreachable; + std.process.exit(0); +} + +const mem = std.mem; +const Allocator = mem.Allocator; + +extern fn roc__mainForHost_1_exposed(i64, *i64) void; + +const Unit = extern struct {}; + +pub export fn main() u8 { + const stdout = std.io.getStdOut().writer(); + const fib_number_to_find: u64 = 10; // find the nth Fibonacci number + const iterations: usize = 50; // number of times to repeatedly find that Fibonacci number + + // make space for the result + var callresult = 0; + var remaining_iterations = iterations; + + while (remaining_iterations > 0) { + // actually call roc to populate the callresult + roc__mainForHost_1_exposed(fib_number_to_find, &callresult); + + remaining_iterations -= 1; + } + + // stdout the final result + stdout.print( + "After calling the Roc app {d} times, the Fibonacci number at index {d} is {d}\n", + .{ iterations, fib_number_to_find, callresult }, + ) catch unreachable; + + return 0; +} diff --git a/roc_std/src/lib.rs b/roc_std/src/lib.rs index d5a15b4627..b6f07dcb8a 100644 --- a/roc_std/src/lib.rs +++ b/roc_std/src/lib.rs @@ -758,69 +758,6 @@ pub enum RocResult { Ok(Ok), } -#[allow(non_camel_case_types)] -type c_char = u8; - -#[repr(u64)] -pub enum RocCallResult { - Success(T), - Failure(*mut c_char), -} - -impl From> for Result { - fn from(call_result: RocCallResult) -> Self { - use RocCallResult::*; - - match call_result { - Success(value) => Ok(value), - Failure(failure) => Err({ - let msg = unsafe { - let mut null_byte_index = 0; - loop { - if *failure.offset(null_byte_index) == 0 { - break; - } - null_byte_index += 1; - } - - let bytes = core::slice::from_raw_parts(failure, null_byte_index as usize); - - core::str::from_utf8_unchecked(bytes) - }; - - msg - }), - } - } -} - -impl<'a, T: Sized + Copy> From<&'a RocCallResult> for Result { - fn from(call_result: &'a RocCallResult) -> Self { - use RocCallResult::*; - - match call_result { - Success(value) => Ok(*value), - Failure(failure) => Err({ - let msg = unsafe { - let mut null_byte_index = 0; - loop { - if *failure.offset(null_byte_index) == 0 { - break; - } - null_byte_index += 1; - } - - let bytes = core::slice::from_raw_parts(*failure, null_byte_index as usize); - - core::str::from_utf8_unchecked(bytes) - }; - - msg - }), - } - } -} - #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub struct RocDec(pub i128); diff --git a/shell.nix b/shell.nix index 0cc46a5998..cfbf59f3fe 100644 --- a/shell.nix +++ b/shell.nix @@ -46,6 +46,7 @@ let python3 llvmPkgs.llvm.dev llvmPkgs.clang + libxkbcommon pkg-config zig