From 4b585cc6c65ae187a852be4b44a0893d3c0f7d00 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 9 Feb 2022 13:44:15 +0000 Subject: [PATCH 01/30] wasm: More explicit todo! statements --- compiler/gen_wasm/src/backend.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index 716758fc56..3e70a287fc 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -578,7 +578,22 @@ impl<'a> WasmBackend<'a> { index, } => self.expr_union_at_index(*structure, *tag_id, union_layout, *index, sym), - _ => todo!("Expression `{}`", expr.to_pretty(100)), + Expr::Reuse { + symbol: _, + update_tag_id: _, + update_mode: _, + tag_layout: _, + tag_name: _, + tag_id: _, + arguments: _, + } => todo!("Expression `{}`", expr.to_pretty(100)), + + Expr::Reset { + symbol: _, + update_mode: _, + } => todo!("Expression `{}`", expr.to_pretty(100)), + + Expr::RuntimeErrorFunction(_) => todo!("Expression `{}`", expr.to_pretty(100)), } } From abfa4b15220a99594023d0eb952efb6f25ca2f4c Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 9 Feb 2022 13:45:25 +0000 Subject: [PATCH 02/30] test_gen: allow non-llvm tests to expect errors --- compiler/test_gen/src/gen_primitives.rs | 4 +--- compiler/test_gen/src/helpers/llvm.rs | 32 +++++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs index 2f97c23753..3fcc8ea143 100644 --- a/compiler/test_gen/src/gen_primitives.rs +++ b/compiler/test_gen/src/gen_primitives.rs @@ -3,8 +3,6 @@ use crate::helpers::llvm::assert_evals_to; #[cfg(feature = "gen-llvm")] use crate::helpers::llvm::assert_expect_failed; #[cfg(feature = "gen-llvm")] -use crate::helpers::llvm::assert_llvm_evals_to; -#[cfg(feature = "gen-llvm")] use crate::helpers::llvm::assert_non_opt_evals_to; #[cfg(feature = "gen-dev")] @@ -2473,7 +2471,7 @@ fn function_malformed_pattern() { #[cfg(any(feature = "gen-llvm"))] #[should_panic(expected = "Hit an erroneous type when creating a layout for")] fn call_invalid_layout() { - assert_llvm_evals_to!( + assert_evals_to!( indoc!( r#" f : I64 -> I64 diff --git a/compiler/test_gen/src/helpers/llvm.rs b/compiler/test_gen/src/helpers/llvm.rs index d9af1a821d..91cad5b6da 100644 --- a/compiler/test_gen/src/helpers/llvm.rs +++ b/compiler/test_gen/src/helpers/llvm.rs @@ -590,19 +590,31 @@ macro_rules! assert_llvm_evals_to { #[allow(unused_macros)] macro_rules! assert_evals_to { ($src:expr, $expected:expr, $ty:ty) => {{ - assert_evals_to!($src, $expected, $ty, $crate::helpers::llvm::identity); + assert_evals_to!($src, $expected, $ty, $crate::helpers::llvm::identity, false); }}; - ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { + ($src:expr, $expected:expr, $ty:ty, $transform:expr) => {{ // same as above, except with an additional transformation argument. - { - #[cfg(feature = "wasm-cli-run")] - $crate::helpers::llvm::assert_wasm_evals_to!( - $src, $expected, $ty, $transform, false, false - ); + assert_evals_to!($src, $expected, $ty, $transform, false); + }}; + ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{ + // same as above, except with ignore_problems. + #[cfg(feature = "wasm-cli-run")] + $crate::helpers::llvm::assert_wasm_evals_to!( + $src, + $expected, + $ty, + $transform, + $ignore_problems + ); - $crate::helpers::llvm::assert_llvm_evals_to!($src, $expected, $ty, $transform, false); - } - }; + $crate::helpers::llvm::assert_llvm_evals_to!( + $src, + $expected, + $ty, + $transform, + $ignore_problems + ); + }}; } #[allow(unused_macros)] From 0d08c97e44d4423bdfd83da04c739b78cee3699e Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 9 Feb 2022 13:46:31 +0000 Subject: [PATCH 03/30] wasm: enable a test that has a Stmt::RuntimeError --- compiler/test_gen/src/gen_primitives.rs | 2 +- compiler/test_gen/src/helpers/wasm.rs | 51 +++++++++---------------- 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/compiler/test_gen/src/gen_primitives.rs b/compiler/test_gen/src/gen_primitives.rs index 3fcc8ea143..360aa4d767 100644 --- a/compiler/test_gen/src/gen_primitives.rs +++ b/compiler/test_gen/src/gen_primitives.rs @@ -2468,7 +2468,7 @@ fn function_malformed_pattern() { } #[test] -#[cfg(any(feature = "gen-llvm"))] +#[cfg(any(feature = "gen-llvm", feature = "gen-wasm"))] #[should_panic(expected = "Hit an erroneous type when creating a layout for")] fn call_invalid_layout() { assert_evals_to!( diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index a0fb311283..c142dce15b 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -178,7 +178,7 @@ fn load_bytes_into_runtime(bytes: Vec) -> wasmer::Instance { } #[allow(dead_code)] -pub fn assert_wasm_evals_to_help(src: &str, phantom: PhantomData) -> Result +pub fn assert_evals_to_help(src: &str, phantom: PhantomData) -> Result where T: FromWasmerMemory + Wasm32Result, { @@ -315,42 +315,31 @@ pub fn debug_memory_hex(memory: &Memory, address: i32, size: usize) { } #[allow(unused_macros)] -macro_rules! assert_wasm_evals_to { +macro_rules! assert_evals_to { + ($src:expr, $expected:expr, $ty:ty) => { + $crate::helpers::wasm::assert_evals_to!( + $src, + $expected, + $ty, + $crate::helpers::wasm::identity, + false + ) + }; + ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { + $crate::helpers::wasm::assert_evals_to!($src, $expected, $ty, $transform, false); + }; + + ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{ let phantom = std::marker::PhantomData; - match $crate::helpers::wasm::assert_wasm_evals_to_help::<$ty>($src, phantom) { - Err(msg) => panic!("{:?}", msg), + let _ = $ignore_problems; // Ignore `ignore_problems`! It is used for LLVM tests. + match $crate::helpers::wasm::assert_evals_to_help::<$ty>($src, phantom) { + Err(msg) => panic!("{}", msg), Ok(actual) => { assert_eq!($transform(actual), $expected) } } - }; - - ($src:expr, $expected:expr, $ty:ty) => { - $crate::helpers::wasm::assert_wasm_evals_to!( - $src, - $expected, - $ty, - $crate::helpers::wasm::identity - ); - }; - - ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { - $crate::helpers::wasm::assert_wasm_evals_to!($src, $expected, $ty, $transform); - }; -} - -#[allow(unused_macros)] -macro_rules! assert_evals_to { - ($src:expr, $expected:expr, $ty:ty) => {{ - assert_evals_to!($src, $expected, $ty, $crate::helpers::wasm::identity); }}; - ($src:expr, $expected:expr, $ty:ty, $transform:expr) => { - // Same as above, except with an additional transformation argument. - { - $crate::helpers::wasm::assert_wasm_evals_to!($src, $expected, $ty, $transform); - } - }; } #[allow(dead_code)] @@ -379,8 +368,6 @@ macro_rules! assert_refcounts { #[allow(unused_imports)] pub(crate) use assert_evals_to; -#[allow(unused_imports)] -pub(crate) use assert_wasm_evals_to; #[allow(unused_imports)] pub(crate) use assert_refcounts; From 8c5fe2ae226e1581dec353cb2448780df4549bb7 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Thu, 10 Feb 2022 10:13:54 +0000 Subject: [PATCH 04/30] wasm: implement Stmt::RuntimeError --- compiler/gen_wasm/src/backend.rs | 51 +++++++++++-------- compiler/gen_wasm/src/wasm_module/sections.rs | 2 +- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index 3e70a287fc..440fdcc96a 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -78,6 +78,9 @@ impl<'a> WasmBackend<'a> { index: STACK_POINTER_GLOBAL_ID, }); + // TODO: Read the actual address ranges of preloaded data sections, in case they do something unexpected + let next_constant_addr = CONST_SEGMENT_BASE_ADDR + module.data.bytes.len() as u32; + WasmBackend { env, interns, @@ -86,7 +89,7 @@ impl<'a> WasmBackend<'a> { module, layout_ids, - next_constant_addr: CONST_SEGMENT_BASE_ADDR, + next_constant_addr, fn_index_offset, called_preload_fns: Vec::with_capacity_in(2, env.arena), proc_symbols, @@ -529,7 +532,23 @@ impl<'a> WasmBackend<'a> { } fn stmt_runtime_error(&mut self, msg: &'a str) { - todo!("RuntimeError {:?}", msg) + // Create a zero-terminated version of the message string + let mut bytes = Vec::with_capacity_in(msg.len() + 1, self.env.arena); + bytes.extend_from_slice(msg.as_bytes()); + bytes.push(0); + + // Store it in the app's data section + let sym = self.create_symbol(msg); + let (linker_sym_index, elements_addr) = self.store_bytes_in_data_section(&bytes, sym); + + // Pass its address to roc_panic + let tag_id = 0; + self.code_builder + .i32_const_mem_addr(elements_addr, linker_sym_index); + self.code_builder.i32_const(tag_id); + self.call_zig_builtin_after_loading_args("roc_panic", 2, false); + + self.code_builder.unreachable_(); } /********************************************************** @@ -540,7 +559,7 @@ impl<'a> WasmBackend<'a> { fn expr(&mut self, sym: Symbol, expr: &Expr<'a>, layout: &Layout<'a>, storage: &StoredValue) { match expr { - Expr::Literal(lit) => self.expr_literal(lit, storage, sym, layout), + Expr::Literal(lit) => self.expr_literal(lit, storage, sym), Expr::Call(roc_mono::ir::Call { call_type, @@ -601,13 +620,7 @@ impl<'a> WasmBackend<'a> { * Literals *******************************************************************/ - fn expr_literal( - &mut self, - lit: &Literal<'a>, - storage: &StoredValue, - sym: Symbol, - layout: &Layout<'a>, - ) { + fn expr_literal(&mut self, lit: &Literal<'a>, storage: &StoredValue, sym: Symbol) { let invalid_error = || internal_error!("Literal value {:?} has invalid storage {:?}", lit, storage); @@ -670,8 +683,9 @@ impl<'a> WasmBackend<'a> { self.code_builder.i64_const(str_as_int); self.code_builder.i64_store(Align::Bytes4, offset); } else { + let bytes = string.as_bytes(); let (linker_sym_index, elements_addr) = - self.expr_literal_big_str(string, sym, layout); + self.store_bytes_in_data_section(bytes, sym); self.code_builder.get_local(local_id); self.code_builder @@ -693,16 +707,11 @@ impl<'a> WasmBackend<'a> { /// Create a string constant in the module data section /// Return the data we need for code gen: linker symbol index and memory address - fn expr_literal_big_str( - &mut self, - string: &'a str, - sym: Symbol, - layout: &Layout<'a>, - ) -> (u32, u32) { + fn store_bytes_in_data_section(&mut self, bytes: &[u8], sym: Symbol) -> (u32, u32) { // Place the segment at a 4-byte aligned offset let segment_addr = round_up_to_alignment!(self.next_constant_addr, PTR_SIZE); let elements_addr = segment_addr + PTR_SIZE; - let length_with_refcount = 4 + string.len(); + let length_with_refcount = 4 + bytes.len(); self.next_constant_addr = segment_addr + length_with_refcount as u32; let mut segment = DataSegment { @@ -713,14 +722,14 @@ impl<'a> WasmBackend<'a> { // Prefix the string bytes with "infinite" refcount let refcount_max_bytes: [u8; 4] = (REFCOUNT_MAX as i32).to_le_bytes(); segment.init.extend_from_slice(&refcount_max_bytes); - segment.init.extend_from_slice(string.as_bytes()); + segment.init.extend_from_slice(bytes); let segment_index = self.module.data.append_segment(segment); // Generate linker symbol let name = self .layout_ids - .get(sym, layout) + .get(sym, &Layout::Builtin(Builtin::Str)) .to_symbol_string(sym, self.interns); let linker_symbol = SymInfo::Data(DataSymbol::Defined { @@ -728,7 +737,7 @@ impl<'a> WasmBackend<'a> { name: name.clone(), segment_index, segment_offset: 4, - size: string.len() as u32, + size: bytes.len() as u32, }); // Ensure the linker keeps the segment aligned when relocating it diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index ea8ca5ec62..29d82bb89b 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -850,7 +850,7 @@ impl Serialize for DataSegment<'_> { #[derive(Debug)] pub struct DataSection<'a> { count: u32, - bytes: Vec<'a, u8>, + pub bytes: Vec<'a, u8>, } impl<'a> DataSection<'a> { From 55465d15a0b062fd9f7c0d709519dc536b52222f Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Thu, 10 Feb 2022 10:15:19 +0000 Subject: [PATCH 05/30] wasm: include Name section in output binary, for debugging --- compiler/gen_wasm/src/wasm_module/mod.rs | 1 + compiler/gen_wasm/src/wasm_module/sections.rs | 42 ++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 2ac492e620..9006d7954e 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -66,6 +66,7 @@ impl<'a> WasmModule<'a> { self.element.serialize(buffer); self.code.serialize(buffer); self.data.serialize(buffer); + self.names.serialize(buffer); } /// Serialize the module to bytes diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 29d82bb89b..a6f7f02640 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -928,8 +928,9 @@ enum NameSubSections { LocalNames = 2, } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct NameSection<'a> { + pub bytes: Vec<'a, u8>, pub functions: MutMap<&'a [u8], u32>, } @@ -938,13 +939,6 @@ impl<'a> NameSection<'a> { const NAME: &'static str = "name"; pub fn parse(arena: &'a Bump, module_bytes: &[u8], cursor: &mut usize) -> Self { - let functions = MutMap::default(); - let mut section = NameSection { functions }; - section.parse_help(arena, module_bytes, cursor); - section - } - - fn parse_help(&mut self, arena: &'a Bump, module_bytes: &[u8], cursor: &mut usize) { // Custom section ID let section_id_byte = module_bytes[*cursor]; if section_id_byte != Self::ID as u8 { @@ -958,16 +952,32 @@ impl<'a> NameSection<'a> { *cursor += 1; // Section size - let section_size = parse_u32_or_panic(module_bytes, cursor); - let section_end = *cursor + section_size as usize; + let section_size = parse_u32_or_panic(module_bytes, cursor) as usize; + let section_end = *cursor + section_size; + let mut bytes = Vec::with_capacity_in(section_size, arena); + bytes.extend_from_slice(&module_bytes[*cursor..section_end]); + let functions = MutMap::default(); + let mut section = NameSection { bytes, functions }; + + section.parse_body(arena, module_bytes, cursor, section_end); + section + } + + fn parse_body( + &mut self, + arena: &'a Bump, + module_bytes: &[u8], + cursor: &mut usize, + section_end: usize, + ) { // Custom section name let section_name_len = parse_u32_or_panic(module_bytes, cursor); let section_name_end = *cursor + section_name_len as usize; let section_name = &module_bytes[*cursor..section_name_end]; if section_name != Self::NAME.as_bytes() { internal_error!( - "Expected Custon section {:?}, found {:?}", + "Expected Custom section {:?}, found {:?}", Self::NAME, std::str::from_utf8(section_name) ); @@ -1008,6 +1018,16 @@ impl<'a> NameSection<'a> { } } +impl<'a> Serialize for NameSection<'a> { + fn serialize(&self, buffer: &mut T) { + if !self.bytes.is_empty() { + let header_indices = write_section_header(buffer, Self::ID); + buffer.append_slice(&self.bytes); + update_section_size(buffer, header_indices); + } + } +} + /******************************************************************* * * Unit tests From 2af4016f3435869dd68bb3fb8578b4fbd3ccb8c3 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Thu, 10 Feb 2022 10:15:59 +0000 Subject: [PATCH 06/30] wasm: improve the HTML debugger's WASI syscall implementations --- compiler/test_gen/src/helpers/debug-wasm-test.html | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/compiler/test_gen/src/helpers/debug-wasm-test.html b/compiler/test_gen/src/helpers/debug-wasm-test.html index caf1b01901..ba92959499 100644 --- a/compiler/test_gen/src/helpers/debug-wasm-test.html +++ b/compiler/test_gen/src/helpers/debug-wasm-test.html @@ -258,6 +258,7 @@ function fd_write(fd, iovs_ptr, iovs_len, nwritten_mut_ptr) { let string_buffer = ""; let nwritten = 0; + const STDOUT = 1; for (let i = 0; i < iovs_len; i++) { const index32 = iovs_ptr >> 2; @@ -282,16 +283,18 @@ } wasiLinkObject.memory32[nwritten_mut_ptr >> 2] = nwritten; if (string_buffer) { - console.log(string_buffer); + if (fd === STDOUT) { + console.log(string_buffer); + } else { + console.error(string_buffer); + } } return 0; } // proc_exit : (i32) -> nil function proc_exit(exit_code) { - if (exit_code) { - throw new Error(`Wasm exited with code ${exit_code}`); - } + throw new Error(`Wasm exited with code ${exit_code}`); } // Signatures from wasm_test_platform.o From 5f64f32a4a15ece0b9414b633d9802e73d7597e4 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 07:33:49 +0000 Subject: [PATCH 07/30] wasm: Better implementation of roc_panic in tests --- compiler/test_gen/src/helpers/wasm_test_platform.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm_test_platform.c b/compiler/test_gen/src/helpers/wasm_test_platform.c index b01543d0de..0e767dbffe 100644 --- a/compiler/test_gen/src/helpers/wasm_test_platform.c +++ b/compiler/test_gen/src/helpers/wasm_test_platform.c @@ -1,4 +1,5 @@ #include +#include // Makes test runs take 50% longer, due to linking #define ENABLE_PRINTF 0 @@ -121,14 +122,13 @@ void roc_dealloc(void *ptr, unsigned int alignment) //-------------------------- -void roc_panic(void *ptr, unsigned int alignment) +void roc_panic(char *msg, unsigned int tag_id) { -#if ENABLE_PRINTF - char *msg = (char *)ptr; - fprintf(stderr, - "Application crashed with message\n\n %s\n\nShutting down\n", msg); -#endif - abort(); + // Note: no dynamic string formatting + fputs("Application crashed with message\n\n ", stderr); + fputs(msg, stderr); + fputs("\n\nShutting down\n", stderr); + exit(101); } //-------------------------- From 57179eb45825e9513f04adf13f41adbd0506ebb3 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 08:12:36 +0000 Subject: [PATCH 08/30] test_gen: comments about wasm panics --- compiler/test_gen/src/helpers/wasm.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index c142dce15b..e235edec53 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -330,9 +330,12 @@ macro_rules! assert_evals_to { $crate::helpers::wasm::assert_evals_to!($src, $expected, $ty, $transform, false); }; - ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{ + // LLVM tests call the last arg $ignore_problems rather than $expect_panic + // Either way, it's only `true` for tests that use `should_panic`. + // In Wasm, `should_panic` only works if we spawn a child process. + ($src:expr, $expected:expr, $ty:ty, $transform:expr, $expect_panic: expr) => {{ let phantom = std::marker::PhantomData; - let _ = $ignore_problems; // Ignore `ignore_problems`! It is used for LLVM tests. + let _ = $expect_panic; // TODO match $crate::helpers::wasm::assert_evals_to_help::<$ty>($src, phantom) { Err(msg) => panic!("{}", msg), Ok(actual) => { From 97c939c191da74761ff33f470e5e61d126a1478c Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 08:16:45 +0000 Subject: [PATCH 09/30] test_gen: reduce wasm test time from 1m30s to 15s using Wasmer's singlepass compiler --- Cargo.lock | 46 ++++++++++++++++++++++++++++++++++++ compiler/test_gen/Cargo.toml | 2 +- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7500ac6277..e25b94e7c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1193,6 +1193,32 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" +[[package]] +name = "dynasm" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b1801e630bd336d0bbbdbf814de6cc749c9a400c7e3d995e6adfd455d0c83c" +dependencies = [ + "bitflags", + "byteorder", + "lazy_static", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "dynasmrt" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d428afc93ad288f6dffc1fa5f4a78201ad2eec33c5a522e51c181009eb09061" +dependencies = [ + "byteorder", + "dynasm", + "memmap2 0.5.0", +] + [[package]] name = "either" version = "1.6.1" @@ -4742,6 +4768,7 @@ dependencies = [ "thiserror", "wasmer-compiler", "wasmer-compiler-cranelift", + "wasmer-compiler-singlepass", "wasmer-derive", "wasmer-engine", "wasmer-engine-dylib", @@ -4790,6 +4817,25 @@ dependencies = [ "wasmer-vm", ] +[[package]] +name = "wasmer-compiler-singlepass" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9429b9f7708c582d855b1787f09c7029ff23fb692550d4a1cc351c8ea84c3014" +dependencies = [ + "byteorder", + "dynasm", + "dynasmrt", + "lazy_static", + "loupe", + "more-asserts", + "rayon", + "smallvec", + "wasmer-compiler", + "wasmer-types", + "wasmer-vm", +] + [[package]] name = "wasmer-derive" version = "2.0.0" diff --git a/compiler/test_gen/Cargo.toml b/compiler/test_gen/Cargo.toml index 607b14b7ff..53e7e968c0 100644 --- a/compiler/test_gen/Cargo.toml +++ b/compiler/test_gen/Cargo.toml @@ -39,7 +39,7 @@ libc = "0.2.106" inkwell = { path = "../../vendor/inkwell" } target-lexicon = "0.12.2" libloading = "0.7.1" -wasmer = { version = "2.0.0", default-features = false, features = ["default-cranelift", "default-universal"] } +wasmer = { version = "2.0.0", default-features = false, features = ["default-singlepass", "default-universal"] } wasmer-wasi = "2.0.0" tempfile = "3.2.0" indoc = "1.0.3" From 54788b035723713046abb7fb95683ce0a9a6cda4 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 18:44:47 +0000 Subject: [PATCH 10/30] wasm: Create a full model of the ElementSection --- compiler/gen_wasm/src/wasm_module/mod.rs | 8 +- compiler/gen_wasm/src/wasm_module/sections.rs | 198 ++++++++++++++++++ 2 files changed, 202 insertions(+), 4 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 9006d7954e..619edfb5ab 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -15,8 +15,8 @@ use crate::wasm_module::serialize::SkipBytes; use self::linking::{LinkingSection, RelocationSection}; use self::sections::{ - CodeSection, DataSection, ExportSection, FunctionSection, GlobalSection, ImportSection, - MemorySection, NameSection, OpaqueSection, Section, SectionId, TypeSection, + CodeSection, DataSection, ElementSection, ExportSection, FunctionSection, GlobalSection, + ImportSection, MemorySection, NameSection, OpaqueSection, Section, SectionId, TypeSection, }; use self::serialize::{SerialBuffer, Serialize}; @@ -32,7 +32,7 @@ pub struct WasmModule<'a> { pub global: GlobalSection<'a>, pub export: ExportSection<'a>, pub start: OpaqueSection<'a>, - pub element: OpaqueSection<'a>, + pub element: ElementSection<'a>, pub code: CodeSection<'a>, pub data: DataSection<'a>, pub names: NameSection<'a>, @@ -143,7 +143,7 @@ impl<'a> WasmModule<'a> { let export = ExportSection::empty(arena); let start = OpaqueSection::preload(SectionId::Start, arena, bytes, &mut cursor); - let element = OpaqueSection::preload(SectionId::Element, arena, bytes, &mut cursor); + let element = ElementSection::preload(arena, bytes, &mut cursor); let code = CodeSection::preload(arena, bytes, &mut cursor, import.function_count); let data = DataSection::preload(arena, bytes, &mut cursor); diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index a6f7f02640..8a42862dcd 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -699,6 +699,204 @@ impl<'a> Serialize for ExportSection<'a> { } } +/******************************************************************* + * + * Element section + * + * Elements are entries in tables (see Table section) + * For example, Wasm uses a function table instead of function pointers, + * and each entry in that function table is an element. + * The call_indirect instruction uses element indices to refer to functions. + * This section therefore enumerates all indirectly-called functions. + * + *******************************************************************/ + +/* +https://webassembly.github.io/spec/core/binary/modules.html#binary-elemsec + +Note: Wasm MVP only had variant 0x00, and tables only contained functions. + +byte fields ⇒ syntax +------------------------------------------------------------------------------------------------------------------------------------- +0x00 e:expr y∗:vec(funcidx) ⇒ {type funcref, init ((ref.func y) end)∗, mode active {table 0, offset e}} +0x01 et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode passive} +0x02 x:tableidx e:expr et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode active {table x, offset e}} +0x03 et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode declarative} + +0x04 e:expr el∗:vec(expr) ⇒ {type funcref, init el∗, mode active {table 0, offset e}} +0x05 et:reftype el∗:vec(expr) ⇒ {type et, init el∗, mode passive} +0x06 x:tableidx e:expr et:reftype el∗:vec(expr) ⇒ {type et, init el∗, mode active {table x, offset e}} +0x07 et:reftype el∗:vec(expr) ⇒ {type et, init el ∗ , mode declarative} + +The initial byte can be interpreted as a bitfield +Bit 0 indicates a passive or declarative segment +Bit 1 indicates the presence of an explicit table index for an active segment and otherwise distinguishes passive from declarative segments +Bit 2 indicates the use of element type and element expressions instead of element kind and element indices. + +Active means "element is loaded into the table on module instantiation" +Passive means it's loaded by explicit instructions in the code section +Declarative is a declaration that a reference will be taken at runtime + +Sigh. This is only the post-MVP version of Wasm! Some day it'll end up as convoluted as x86, wait and see. +*/ + +#[repr(u8)] +#[derive(Debug)] +#[allow(dead_code)] +enum ElementKind { + FuncRef = 0, +} + +#[repr(u8)] +#[allow(dead_code)] +enum ElementSegmentFormatId { + ActiveImplicitTableIndex = 0x00, + PassiveKindAndIndex = 0x01, + ActiveExplicitTableKindAndIndex = 0x02, + DeclarativeKindAndIndex = 0x03, + ActiveImplicitTableTypeAndExpr = 0x04, + PassiveTypeAndExpr = 0x05, + ActiveExplicitTableTypeAndExpr = 0x06, + DeclarativeTypeAndExpr = 0x07, +} + +// A representation based on the (convoluted) binary format. +// NOTE: If we ever need a more intuitive format, we can base it on the syntax doc +// https://webassembly.github.io/spec/core/syntax/modules.html#syntax-elem +#[derive(Debug)] +#[allow(dead_code)] +enum ElementSegment<'a> { + /// This is the only variant we currently use. It is from the original Wasm MVP. + /// The rest are dead code but kept in case we need them later. + ActiveImplicitTableIndex { + offset: ConstExpr, + fn_indices: Vec<'a, u32>, + }, + PassiveKindAndIndex { + kind: ElementKind, + fn_indices: Vec<'a, u32>, + }, + ActiveExplicitTableKindAndIndex { + table_idx: u32, + offset: ConstExpr, + kind: ElementKind, + fn_indices: Vec<'a, u32>, + }, + DeclarativeKindAndIndex { + kind: ElementKind, + fn_indices: Vec<'a, u32>, + }, + ActiveImplicitTableTypeAndExpr { + offset: ConstExpr, + init: Vec<'a, ConstExpr>, + }, + PassiveTypeAndExpr { + elem_type: RefType, + init: Vec<'a, ConstExpr>, + }, + ActiveExplicitTableTypeAndExpr { + table_idx: u32, + offset: ConstExpr, + elem_type: RefType, + init: Vec<'a, ConstExpr>, + }, + DeclarativeTypeAndExpr { + elem_type: RefType, + init: Vec<'a, ConstExpr>, + }, +} + +impl<'a> ElementSegment<'a> { + fn parse(arena: &'a Bump, bytes: &[u8], cursor: &mut usize) -> Self { + // In practice we only need the original MVP format + let format_id = bytes[*cursor]; + assert!(format_id == ElementSegmentFormatId::ActiveImplicitTableIndex as u8); + *cursor += 1; + + // The table index offset is encoded as a ConstExpr, but only I32 makes sense + let const_expr_opcode = bytes[*cursor]; + assert!(const_expr_opcode == OpCode::I32CONST as u8); + *cursor += 1; + let offset = parse_u32_or_panic(bytes, cursor); + assert!(bytes[*cursor] == OpCode::END as u8); + *cursor += 1; + + let num_elems = parse_u32_or_panic(bytes, cursor); + let mut fn_indices = Vec::with_capacity_in(num_elems as usize, arena); + for _ in 0..num_elems { + let fn_idx = parse_u32_or_panic(bytes, cursor); + + fn_indices.push(fn_idx); + } + + ElementSegment::ActiveImplicitTableIndex { + offset: ConstExpr::I32(offset as i32), + fn_indices, + } + } + + fn size(&self) -> usize { + let variant_id = 1; + let constexpr_opcode = 1; + let constexpr_value = MAX_SIZE_ENCODED_U32; + let vec_len = MAX_SIZE_ENCODED_U32; + let vec_contents = MAX_SIZE_ENCODED_U32 + * if let ElementSegment::ActiveImplicitTableIndex { fn_indices, .. } = &self { + fn_indices.len() + } else { + internal_error!("Unsupported ElementSegment {:?}", self) + }; + variant_id + constexpr_opcode + constexpr_value + vec_len + vec_contents + } +} + +impl<'a> Serialize for ElementSegment<'a> { + fn serialize(&self, buffer: &mut T) { + if let ElementSegment::ActiveImplicitTableIndex { offset, fn_indices } = &self { + buffer.append_u8(ElementSegmentFormatId::ActiveImplicitTableIndex as u8); + offset.serialize(buffer); + fn_indices.serialize(buffer); + } else { + internal_error!("Unsupported ElementSegment {:?}", self) + } + } +} + +#[derive(Debug)] +pub struct ElementSection<'a> { + segments: Vec<'a, ElementSegment<'a>>, +} + +impl<'a> ElementSection<'a> { + const ID: SectionId = SectionId::Element; + + pub fn preload(arena: &'a Bump, module_bytes: &[u8], cursor: &mut usize) -> Self { + let (num_segments, body_bytes) = parse_section(Self::ID, module_bytes, cursor); + + let mut segments = Vec::with_capacity_in(num_segments as usize, arena); + + let mut body_cursor = 0; + for _ in 0..num_segments { + let seg = ElementSegment::parse(arena, body_bytes, &mut body_cursor); + segments.push(seg); + } + + ElementSection { segments } + } + + pub fn size(&self) -> usize { + self.segments.iter().map(|seg| seg.size()).sum() + } +} + +impl<'a> Serialize for ElementSection<'a> { + fn serialize(&self, buffer: &mut T) { + let header_indices = write_section_header(buffer, Self::ID); + self.segments.serialize(buffer); + update_section_size(buffer, header_indices); + } +} + /******************************************************************* * * Code section (see also code_builder.rs) From fcda6fabe25148e4fe4ce0ed81be6d24904ce985 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 18:46:18 +0000 Subject: [PATCH 11/30] wasm: replace a panic! with an internal_error! --- compiler/gen_wasm/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/gen_wasm/src/lib.rs b/compiler/gen_wasm/src/lib.rs index 7030419e49..8170a345f3 100644 --- a/compiler/gen_wasm/src/lib.rs +++ b/compiler/gen_wasm/src/lib.rs @@ -213,7 +213,7 @@ macro_rules! round_up_to_alignment { if $alignment_bytes <= 1 { $unaligned } else if $alignment_bytes.count_ones() != 1 { - panic!( + internal_error!( "Cannot align to {} bytes. Not a power of 2.", $alignment_bytes ); From 63c33d82e38d7bc8f7ec4da2d089175489780219 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Sat, 12 Feb 2022 00:09:35 +0000 Subject: [PATCH 12/30] wasm: Improve dead code elimination to handle indirect calls --- .../gen_wasm/src/wasm_module/dead_code.rs | 33 +++++++++--- compiler/gen_wasm/src/wasm_module/mod.rs | 21 +++++++- compiler/gen_wasm/src/wasm_module/sections.rs | 51 ++++++++++++++----- 3 files changed, 83 insertions(+), 22 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/dead_code.rs b/compiler/gen_wasm/src/wasm_module/dead_code.rs index 382f306322..946b84cf84 100644 --- a/compiler/gen_wasm/src/wasm_module/dead_code.rs +++ b/compiler/gen_wasm/src/wasm_module/dead_code.rs @@ -39,16 +39,16 @@ pub struct PreloadsCallGraph<'a> { } impl<'a> PreloadsCallGraph<'a> { - pub fn new(arena: &'a Bump, import_fn_count: u32, fn_count: u32) -> Self { - let num_preloads = (import_fn_count + fn_count) as usize; + pub fn new(arena: &'a Bump, import_fn_count: usize, fn_count: usize) -> Self { + let num_preloads = import_fn_count + fn_count; let mut code_offsets = Vec::with_capacity_in(num_preloads, arena); let calls = Vec::with_capacity_in(2 * num_preloads, arena); let mut calls_offsets = Vec::with_capacity_in(1 + num_preloads, arena); // Imported functions have zero code length and no calls - code_offsets.extend(std::iter::repeat(0).take(import_fn_count as usize)); - calls_offsets.extend(std::iter::repeat(0).take(import_fn_count as usize)); + code_offsets.extend(std::iter::repeat(0).take(import_fn_count)); + calls_offsets.extend(std::iter::repeat(0).take(import_fn_count)); PreloadsCallGraph { num_preloads, @@ -65,11 +65,18 @@ impl<'a> PreloadsCallGraph<'a> { /// use this backend without a linker. pub fn parse_preloads_call_graph<'a>( arena: &'a Bump, - fn_count: u32, code_section_body: &[u8], - import_fn_count: u32, + import_signatures: &[u32], + function_signatures: &[u32], + indirect_callees: &[u32], ) -> PreloadsCallGraph<'a> { - let mut call_graph = PreloadsCallGraph::new(arena, import_fn_count, fn_count); + let mut call_graph = + PreloadsCallGraph::new(arena, import_signatures.len(), function_signatures.len()); + + let mut signatures = + Vec::with_capacity_in(import_signatures.len() + function_signatures.len(), arena); + signatures.extend_from_slice(import_signatures); + signatures.extend_from_slice(function_signatures); // Iterate over the bytes of the Code section let mut cursor: usize = 0; @@ -88,13 +95,23 @@ pub fn parse_preloads_call_graph<'a>( cursor += 1; // ValueType } - // Parse `call` instructions and skip over all other instructions + // Parse `call` and `call_indirect` instructions, skip over everything else while cursor < func_end { let opcode_byte: u8 = code_section_body[cursor]; if opcode_byte == OpCode::CALL as u8 { cursor += 1; let call_index = parse_u32_or_panic(code_section_body, &mut cursor); call_graph.calls.push(call_index as u32); + } else if opcode_byte == OpCode::CALLINDIRECT as u8 { + cursor += 1; + // Insert all indirect callees with a matching type signature + let sig = parse_u32_or_panic(code_section_body, &mut cursor); + call_graph.calls.extend( + indirect_callees + .iter() + .filter(|f| signatures[**f as usize] == sig), + ); + u32::skip_bytes(code_section_body, &mut cursor); // table_idx } else { OpCode::skip_bytes(code_section_body, &mut cursor); } diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 619edfb5ab..4badf3aefc 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -133,18 +133,35 @@ impl<'a> WasmModule<'a> { let mut types = TypeSection::preload(arena, bytes, &mut cursor); types.parse_offsets(); - let import = ImportSection::preload(arena, bytes, &mut cursor); + let mut import = ImportSection::preload(arena, bytes, &mut cursor); + let import_signatures = import.parse(arena); + let function = FunctionSection::preload(arena, bytes, &mut cursor); + let function_signatures = function.parse(arena); + let table = OpaqueSection::preload(SectionId::Table, arena, bytes, &mut cursor); + let memory = MemorySection::preload(arena, bytes, &mut cursor); + let global = GlobalSection::preload(arena, bytes, &mut cursor); ExportSection::skip_bytes(bytes, &mut cursor); + let export = ExportSection::empty(arena); let start = OpaqueSection::preload(SectionId::Start, arena, bytes, &mut cursor); + let element = ElementSection::preload(arena, bytes, &mut cursor); - let code = CodeSection::preload(arena, bytes, &mut cursor, import.function_count); + let indirect_callees = element.indirect_callees(arena); + + let code = CodeSection::preload( + arena, + bytes, + &mut cursor, + &import_signatures, + &function_signatures, + &indirect_callees, + ); let data = DataSection::preload(arena, bytes, &mut cursor); diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 8a42862dcd..f49ce674e7 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -377,8 +377,8 @@ impl<'a> ImportSection<'a> { self.count += 1; } - fn update_function_count(&mut self) { - let mut f_count = 0; + pub fn parse(&mut self, arena: &'a Bump) -> Vec<'a, u32> { + let mut fn_signatures = bumpalo::vec![in arena]; let mut cursor = 0; while cursor < self.bytes.len() { String::skip_bytes(&self.bytes, &mut cursor); @@ -389,8 +389,7 @@ impl<'a> ImportSection<'a> { match type_id { ImportTypeId::Func => { - f_count += 1; - u32::skip_bytes(&self.bytes, &mut cursor); + fn_signatures.push(parse_u32_or_panic(&self.bytes, &mut cursor)); } ImportTypeId::Table => { TableType::skip_bytes(&self.bytes, &mut cursor); @@ -404,17 +403,16 @@ impl<'a> ImportSection<'a> { } } - self.function_count = f_count; + self.function_count = fn_signatures.len() as u32; + fn_signatures } pub fn from_count_and_bytes(count: u32, bytes: Vec<'a, u8>) -> Self { - let mut created = ImportSection { + ImportSection { bytes, count, function_count: 0, - }; - created.update_function_count(); - created + } } } @@ -442,6 +440,16 @@ impl<'a> FunctionSection<'a> { self.bytes.encode_u32(sig_id); self.count += 1; } + + pub fn parse(&self, arena: &'a Bump) -> Vec<'a, u32> { + let count = self.count as usize; + let mut signatures = Vec::with_capacity_in(count, arena); + let mut cursor = 0; + for _ in 0..count { + signatures.push(parse_u32_or_panic(&self.bytes, &mut cursor)); + } + signatures + } } section_impl!(FunctionSection, SectionId::Function); @@ -887,6 +895,18 @@ impl<'a> ElementSection<'a> { pub fn size(&self) -> usize { self.segments.iter().map(|seg| seg.size()).sum() } + + pub fn indirect_callees(&self, arena: &'a Bump) -> Vec<'a, u32> { + let mut result = bumpalo::vec![in arena]; + for segment in self.segments.iter() { + if let ElementSegment::ActiveImplicitTableIndex { fn_indices, .. } = segment { + result.extend_from_slice(fn_indices); + } else { + internal_error!("Unsupported ElementSegment {:?}", self) + } + } + result + } } impl<'a> Serialize for ElementSection<'a> { @@ -940,14 +960,21 @@ impl<'a> CodeSection<'a> { arena: &'a Bump, module_bytes: &[u8], cursor: &mut usize, - import_fn_count: u32, + import_signatures: &[u32], + function_signatures: &[u32], + indirect_callees: &[u32], ) -> Self { let (preloaded_count, initial_bytes) = parse_section(SectionId::Code, module_bytes, cursor); let preloaded_bytes = arena.alloc_slice_copy(initial_bytes); // TODO: Try to move this call_graph preparation to platform build time - let dead_code_metadata = - parse_preloads_call_graph(arena, preloaded_count, initial_bytes, import_fn_count); + let dead_code_metadata = parse_preloads_call_graph( + arena, + initial_bytes, + import_signatures, + function_signatures, + indirect_callees, + ); CodeSection { preloaded_count, From a9df6f4ff94f21ac5b4d56e910cc2a2dc1e5a997 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 11 Feb 2022 07:35:31 +0000 Subject: [PATCH 13/30] test_gen: create a getter for wasm panic messages --- compiler/test_gen/src/helpers/wasm.rs | 44 ++++++++++++++++++- .../test_gen/src/helpers/wasm_test_platform.c | 8 ++++ 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index e235edec53..54e8f48f7f 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -18,6 +18,7 @@ const OUT_DIR_VAR: &str = "TEST_GEN_OUT"; const TEST_WRAPPER_NAME: &str = "test_wrapper"; const INIT_REFCOUNT_NAME: &str = "init_refcount_test"; +const GET_PANIC_MSG_NAME: &str = "get_panic_msg"; fn promote_expr_to_module(src: &str) -> String { let mut buffer = String::from("app \"test\" provides [ main ] to \"./platform\"\n\nmain =\n"); @@ -136,6 +137,15 @@ fn compile_roc_to_wasm_bytes<'a, T: Wasm32Result>( index: init_refcount_idx, }); + // Export the getter function for panic messages + let get_panic_msg_bytes = GET_PANIC_MSG_NAME.as_bytes(); + let get_panic_msg_idx = module.names.functions[get_panic_msg_bytes]; + module.export.append(Export { + name: arena.alloc_slice_copy(get_panic_msg_bytes), + ty: ExportType::Func, + index: get_panic_msg_idx, + }); + module.remove_dead_preloads(env.arena, called_preload_fns); let mut app_module_bytes = std::vec::Vec::with_capacity(module.size()); @@ -194,7 +204,10 @@ where let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap(); match test_wrapper.call(&[]) { - Err(e) => Err(format!("{:?}", e)), + Err(e) => match get_panic_msg(&instance, memory) { + Ok(msg) => Err(msg), + Err(_) => Err(format!("{}", e)), + }, Ok(result) => { let address = match result[0] { wasmer::Value::I32(a) => a, @@ -218,6 +231,35 @@ where } } +/// Call our test platform's getter function for panic messages +fn get_panic_msg(instance: &wasmer::Instance, memory: &wasmer::Memory) -> Result { + let msg_getter = instance + .exports + .get_function(GET_PANIC_MSG_NAME) + .map_err(|e| format!("{:?}", e))?; + + let msg_result = msg_getter.call(&[]).map_err(|e| format!("{:?}", e))?; + + let msg_addr: i32 = match msg_result[0] { + wasmer::Value::I32(a) => a, + _ => panic!(), + }; + if msg_addr == 0 { + return Err("no panic msg".to_string()); + } + + let msg_index = msg_addr as usize; + let memory_bytes = unsafe { memory.data_unchecked() }; + let msg_len = memory_bytes[msg_index..] + .iter() + .position(|c| *c == 0) + .unwrap(); + + let msg_bytes = &memory_bytes[msg_index..msg_len]; + let msg = unsafe { String::from_utf8_unchecked(msg_bytes.to_vec()) }; + Ok(msg) +} + #[allow(dead_code)] pub fn assert_wasm_refcounts_help( src: &str, diff --git a/compiler/test_gen/src/helpers/wasm_test_platform.c b/compiler/test_gen/src/helpers/wasm_test_platform.c index 0e767dbffe..c67853604e 100644 --- a/compiler/test_gen/src/helpers/wasm_test_platform.c +++ b/compiler/test_gen/src/helpers/wasm_test_platform.c @@ -122,8 +122,16 @@ void roc_dealloc(void *ptr, unsigned int alignment) //-------------------------- +// Allow the test to probe the panic message +char* panic_msg; +char* get_panic_msg() { + return panic_msg; +} + void roc_panic(char *msg, unsigned int tag_id) { + panic_msg = msg; + // Note: no dynamic string formatting fputs("Application crashed with message\n\n ", stderr); fputs(msg, stderr); From b46690ecf2bdee9b1db531a02b07d6f0322f45fc Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Sun, 13 Feb 2022 12:42:50 +0000 Subject: [PATCH 14/30] wasm: Custom Debug impl for NameSection --- compiler/gen_wasm/src/wasm_module/mod.rs | 2 +- compiler/gen_wasm/src/wasm_module/sections.rs | 28 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 4badf3aefc..11722bf0d3 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -119,6 +119,7 @@ impl<'a> WasmModule<'a> { + self.element.size() + self.code.size() + self.data.size() + + self.names.size() } pub fn preload(arena: &'a Bump, bytes: &[u8]) -> Self { @@ -146,7 +147,6 @@ impl<'a> WasmModule<'a> { let global = GlobalSection::preload(arena, bytes, &mut cursor); ExportSection::skip_bytes(bytes, &mut cursor); - let export = ExportSection::empty(arena); let start = OpaqueSection::preload(SectionId::Start, arena, bytes, &mut cursor); diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index f49ce674e7..68e39cbd45 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -1,3 +1,5 @@ +use std::fmt::Debug; + use bumpalo::collections::vec::Vec; use bumpalo::Bump; use roc_collections::all::MutMap; @@ -1153,7 +1155,6 @@ enum NameSubSections { LocalNames = 2, } -#[derive(Debug)] pub struct NameSection<'a> { pub bytes: Vec<'a, u8>, pub functions: MutMap<&'a [u8], u32>, @@ -1189,6 +1190,10 @@ impl<'a> NameSection<'a> { section } + pub fn size(&self) -> usize { + self.bytes.len() + } + fn parse_body( &mut self, arena: &'a Bump, @@ -1253,6 +1258,27 @@ impl<'a> Serialize for NameSection<'a> { } } +impl<'a> Debug for NameSection<'a> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "NameSection\n")?; + + // We want to display index->name because it matches the binary format and looks nicer. + // But our hashmap is name->index because that's what code gen wants to look up. + let mut by_index = std::vec::Vec::with_capacity(self.functions.len()); + for (name, index) in self.functions.iter() { + by_index.push((*index, name)); + } + by_index.sort_unstable(); + + for (index, name) in by_index.iter() { + let name_str = unsafe { std::str::from_utf8_unchecked(name) }; + write!(f, " {:4}: {}\n", index, name_str)?; + } + + Ok(()) + } +} + /******************************************************************* * * Unit tests From 4c7be277c258b28521771fd192fc346c9be37a24 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Mon, 14 Feb 2022 08:45:18 +0000 Subject: [PATCH 15/30] wasm: Keep exported global variables from the preloaded object file --- compiler/gen_wasm/src/wasm_module/mod.rs | 5 +- compiler/gen_wasm/src/wasm_module/sections.rs | 49 +++++++++++++++++-- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 11722bf0d3..818c509c4d 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -11,8 +11,6 @@ pub use linking::SymInfo; use roc_error_macros::internal_error; pub use sections::{ConstExpr, Export, ExportType, Global, GlobalType, Signature}; -use crate::wasm_module::serialize::SkipBytes; - use self::linking::{LinkingSection, RelocationSection}; use self::sections::{ CodeSection, DataSection, ElementSection, ExportSection, FunctionSection, GlobalSection, @@ -146,8 +144,7 @@ impl<'a> WasmModule<'a> { let global = GlobalSection::preload(arena, bytes, &mut cursor); - ExportSection::skip_bytes(bytes, &mut cursor); - let export = ExportSection::empty(arena); + let export = ExportSection::preload_globals(arena, bytes, &mut cursor); let start = OpaqueSection::preload(SectionId::Start, arena, bytes, &mut cursor); diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 68e39cbd45..673e943dcf 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -646,6 +646,18 @@ pub enum ExportType { Global = 3, } +impl From for ExportType { + fn from(x: u8) -> Self { + match x { + 0 => Self::Func, + 1 => Self::Table, + 2 => Self::Mem, + 3 => Self::Global, + _ => internal_error!("invalid ExportType {:2x?}", x), + } + } +} + #[derive(Debug)] pub struct Export<'a> { pub name: &'a [u8], @@ -653,6 +665,19 @@ pub struct Export<'a> { pub index: u32, } +impl<'a> Export<'a> { + fn parse_type(bytes: &[u8], cursor: &mut usize) -> ExportType { + String::skip_bytes(bytes, cursor); // name + + let ty = ExportType::from(bytes[*cursor]); + *cursor += 1; + + u32::skip_bytes(bytes, cursor); // index + + ty + } +} + impl Serialize for Export<'_> { fn serialize(&self, buffer: &mut T) { self.name.serialize(buffer); @@ -683,18 +708,32 @@ impl<'a> ExportSection<'a> { section_size(&self.bytes) } - pub fn empty(arena: &'a Bump) -> Self { + fn empty(arena: &'a Bump) -> Self { ExportSection { count: 0, bytes: Vec::with_capacity_in(256, arena), function_indices: Vec::with_capacity_in(4, arena), } } -} -impl SkipBytes for ExportSection<'_> { - fn skip_bytes(bytes: &[u8], cursor: &mut usize) { - parse_section(Self::ID, bytes, cursor); + /// Preload from object file. Keep only the Global exports, ignore the rest. + pub fn preload_globals(arena: &'a Bump, module_bytes: &[u8], cursor: &mut usize) -> Self { + let (num_exports, body_bytes) = parse_section(Self::ID, module_bytes, cursor); + + let mut export_section = ExportSection::empty(arena); + + let mut body_cursor = 0; + for _ in 0..num_exports { + let export_start = body_cursor; + let export_type = Export::parse_type(body_bytes, &mut body_cursor); + if matches!(export_type, ExportType::Global) { + let global_bytes = &body_bytes[export_start..body_cursor]; + export_section.bytes.extend_from_slice(global_bytes); + export_section.count += 1; + } + } + + export_section } } From 7958158d89ac576a8a929da3a87a66b42888352a Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Sun, 27 Feb 2022 23:42:05 +0000 Subject: [PATCH 16/30] wasm: comments & renaming --- compiler/gen_wasm/src/backend.rs | 21 ++++--------------- .../gen_wasm/src/wasm_module/dead_code.rs | 13 ++++++------ compiler/gen_wasm/src/wasm_module/mod.rs | 8 +++---- compiler/gen_wasm/src/wasm_module/sections.rs | 4 ++-- 4 files changed, 17 insertions(+), 29 deletions(-) diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index 55940bf8f6..b0901f8c46 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -78,7 +78,7 @@ impl<'a> WasmBackend<'a> { index: STACK_POINTER_GLOBAL_ID, }); - // TODO: Read the actual address ranges of preloaded data sections, in case they do something unexpected + // TODO: Examine the actual address ranges of every preloaded data segment in case they are not simply sequential let next_constant_addr = CONST_SEGMENT_BASE_ADDR + module.data.bytes.len() as u32; WasmBackend { @@ -598,22 +598,9 @@ impl<'a> WasmBackend<'a> { index, } => self.expr_union_at_index(*structure, *tag_id, union_layout, *index, sym), - Expr::Reuse { - symbol: _, - update_tag_id: _, - update_mode: _, - tag_layout: _, - tag_name: _, - tag_id: _, - arguments: _, - } => todo!("Expression `{}`", expr.to_pretty(100)), - - Expr::Reset { - symbol: _, - update_mode: _, - } => todo!("Expression `{}`", expr.to_pretty(100)), - - Expr::RuntimeErrorFunction(_) => todo!("Expression `{}`", expr.to_pretty(100)), + Expr::Reuse { .. } | Expr::Reset { .. } | Expr::RuntimeErrorFunction(_) => { + todo!("Expression `{}`", expr.to_pretty(100)) + } } } diff --git a/compiler/gen_wasm/src/wasm_module/dead_code.rs b/compiler/gen_wasm/src/wasm_module/dead_code.rs index 946b84cf84..ecbb59b895 100644 --- a/compiler/gen_wasm/src/wasm_module/dead_code.rs +++ b/compiler/gen_wasm/src/wasm_module/dead_code.rs @@ -66,17 +66,18 @@ impl<'a> PreloadsCallGraph<'a> { pub fn parse_preloads_call_graph<'a>( arena: &'a Bump, code_section_body: &[u8], - import_signatures: &[u32], - function_signatures: &[u32], + imported_fn_signatures: &[u32], + defined_fn_signatures: &[u32], indirect_callees: &[u32], ) -> PreloadsCallGraph<'a> { let mut call_graph = - PreloadsCallGraph::new(arena, import_signatures.len(), function_signatures.len()); + PreloadsCallGraph::new(arena, imported_fn_signatures.len(), defined_fn_signatures.len()); + // Function type signatures, used for indirect calls let mut signatures = - Vec::with_capacity_in(import_signatures.len() + function_signatures.len(), arena); - signatures.extend_from_slice(import_signatures); - signatures.extend_from_slice(function_signatures); + Vec::with_capacity_in(imported_fn_signatures.len() + defined_fn_signatures.len(), arena); + signatures.extend_from_slice(imported_fn_signatures); + signatures.extend_from_slice(defined_fn_signatures); // Iterate over the bytes of the Code section let mut cursor: usize = 0; diff --git a/compiler/gen_wasm/src/wasm_module/mod.rs b/compiler/gen_wasm/src/wasm_module/mod.rs index 818c509c4d..ce89c4ee07 100644 --- a/compiler/gen_wasm/src/wasm_module/mod.rs +++ b/compiler/gen_wasm/src/wasm_module/mod.rs @@ -133,10 +133,10 @@ impl<'a> WasmModule<'a> { types.parse_offsets(); let mut import = ImportSection::preload(arena, bytes, &mut cursor); - let import_signatures = import.parse(arena); + let imported_fn_signatures = import.parse(arena); let function = FunctionSection::preload(arena, bytes, &mut cursor); - let function_signatures = function.parse(arena); + let defined_fn_signatures = function.parse(arena); let table = OpaqueSection::preload(SectionId::Table, arena, bytes, &mut cursor); @@ -155,8 +155,8 @@ impl<'a> WasmModule<'a> { arena, bytes, &mut cursor, - &import_signatures, - &function_signatures, + &imported_fn_signatures, + &defined_fn_signatures, &indirect_callees, ); diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 673e943dcf..431c71916b 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -383,8 +383,8 @@ impl<'a> ImportSection<'a> { let mut fn_signatures = bumpalo::vec![in arena]; let mut cursor = 0; while cursor < self.bytes.len() { - String::skip_bytes(&self.bytes, &mut cursor); - String::skip_bytes(&self.bytes, &mut cursor); + String::skip_bytes(&self.bytes, &mut cursor); // import namespace + String::skip_bytes(&self.bytes, &mut cursor); // import name let type_id = ImportTypeId::from(self.bytes[cursor]); cursor += 1; From 4e7c1fe5e15c1f46dbee84bafc0a965b1b7f5b4e Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Sun, 27 Feb 2022 23:52:25 +0000 Subject: [PATCH 17/30] wasm: simplify ElementSection --- compiler/gen_wasm/src/wasm_module/sections.rs | 124 +++--------------- 1 file changed, 15 insertions(+), 109 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 431c71916b..99b445e100 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -760,114 +760,33 @@ impl<'a> Serialize for ExportSection<'a> { * *******************************************************************/ -/* -https://webassembly.github.io/spec/core/binary/modules.html#binary-elemsec - -Note: Wasm MVP only had variant 0x00, and tables only contained functions. - -byte fields ⇒ syntax -------------------------------------------------------------------------------------------------------------------------------------- -0x00 e:expr y∗:vec(funcidx) ⇒ {type funcref, init ((ref.func y) end)∗, mode active {table 0, offset e}} -0x01 et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode passive} -0x02 x:tableidx e:expr et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode active {table x, offset e}} -0x03 et:elemkind y∗:vec(funcidx) ⇒ {type et, init ((ref.func y) end)∗, mode declarative} - -0x04 e:expr el∗:vec(expr) ⇒ {type funcref, init el∗, mode active {table 0, offset e}} -0x05 et:reftype el∗:vec(expr) ⇒ {type et, init el∗, mode passive} -0x06 x:tableidx e:expr et:reftype el∗:vec(expr) ⇒ {type et, init el∗, mode active {table x, offset e}} -0x07 et:reftype el∗:vec(expr) ⇒ {type et, init el ∗ , mode declarative} - -The initial byte can be interpreted as a bitfield -Bit 0 indicates a passive or declarative segment -Bit 1 indicates the presence of an explicit table index for an active segment and otherwise distinguishes passive from declarative segments -Bit 2 indicates the use of element type and element expressions instead of element kind and element indices. - -Active means "element is loaded into the table on module instantiation" -Passive means it's loaded by explicit instructions in the code section -Declarative is a declaration that a reference will be taken at runtime - -Sigh. This is only the post-MVP version of Wasm! Some day it'll end up as convoluted as x86, wait and see. -*/ - #[repr(u8)] -#[derive(Debug)] -#[allow(dead_code)] -enum ElementKind { - FuncRef = 0, -} - -#[repr(u8)] -#[allow(dead_code)] enum ElementSegmentFormatId { + /// Currently only supporting the original Wasm MVP format since it's the only one in wide use. + /// There are newer formats for other table types, with complex encodings to preserve backward compatibility + /// (Already going down the same path as x86!) ActiveImplicitTableIndex = 0x00, - PassiveKindAndIndex = 0x01, - ActiveExplicitTableKindAndIndex = 0x02, - DeclarativeKindAndIndex = 0x03, - ActiveImplicitTableTypeAndExpr = 0x04, - PassiveTypeAndExpr = 0x05, - ActiveExplicitTableTypeAndExpr = 0x06, - DeclarativeTypeAndExpr = 0x07, } -// A representation based on the (convoluted) binary format. -// NOTE: If we ever need a more intuitive format, we can base it on the syntax doc -// https://webassembly.github.io/spec/core/syntax/modules.html#syntax-elem #[derive(Debug)] -#[allow(dead_code)] -enum ElementSegment<'a> { - /// This is the only variant we currently use. It is from the original Wasm MVP. - /// The rest are dead code but kept in case we need them later. - ActiveImplicitTableIndex { - offset: ConstExpr, - fn_indices: Vec<'a, u32>, - }, - PassiveKindAndIndex { - kind: ElementKind, - fn_indices: Vec<'a, u32>, - }, - ActiveExplicitTableKindAndIndex { - table_idx: u32, - offset: ConstExpr, - kind: ElementKind, - fn_indices: Vec<'a, u32>, - }, - DeclarativeKindAndIndex { - kind: ElementKind, - fn_indices: Vec<'a, u32>, - }, - ActiveImplicitTableTypeAndExpr { - offset: ConstExpr, - init: Vec<'a, ConstExpr>, - }, - PassiveTypeAndExpr { - elem_type: RefType, - init: Vec<'a, ConstExpr>, - }, - ActiveExplicitTableTypeAndExpr { - table_idx: u32, - offset: ConstExpr, - elem_type: RefType, - init: Vec<'a, ConstExpr>, - }, - DeclarativeTypeAndExpr { - elem_type: RefType, - init: Vec<'a, ConstExpr>, - }, +struct ElementSegment<'a> { + offset: ConstExpr, + fn_indices: Vec<'a, u32>, } impl<'a> ElementSegment<'a> { fn parse(arena: &'a Bump, bytes: &[u8], cursor: &mut usize) -> Self { // In practice we only need the original MVP format let format_id = bytes[*cursor]; - assert!(format_id == ElementSegmentFormatId::ActiveImplicitTableIndex as u8); + debug_assert!(format_id == ElementSegmentFormatId::ActiveImplicitTableIndex as u8); *cursor += 1; // The table index offset is encoded as a ConstExpr, but only I32 makes sense let const_expr_opcode = bytes[*cursor]; - assert!(const_expr_opcode == OpCode::I32CONST as u8); + debug_assert!(const_expr_opcode == OpCode::I32CONST as u8); *cursor += 1; let offset = parse_u32_or_panic(bytes, cursor); - assert!(bytes[*cursor] == OpCode::END as u8); + debug_assert!(bytes[*cursor] == OpCode::END as u8); *cursor += 1; let num_elems = parse_u32_or_panic(bytes, cursor); @@ -878,7 +797,7 @@ impl<'a> ElementSegment<'a> { fn_indices.push(fn_idx); } - ElementSegment::ActiveImplicitTableIndex { + ElementSegment { offset: ConstExpr::I32(offset as i32), fn_indices, } @@ -889,25 +808,16 @@ impl<'a> ElementSegment<'a> { let constexpr_opcode = 1; let constexpr_value = MAX_SIZE_ENCODED_U32; let vec_len = MAX_SIZE_ENCODED_U32; - let vec_contents = MAX_SIZE_ENCODED_U32 - * if let ElementSegment::ActiveImplicitTableIndex { fn_indices, .. } = &self { - fn_indices.len() - } else { - internal_error!("Unsupported ElementSegment {:?}", self) - }; + let vec_contents = MAX_SIZE_ENCODED_U32 * self.fn_indices.len(); variant_id + constexpr_opcode + constexpr_value + vec_len + vec_contents } } impl<'a> Serialize for ElementSegment<'a> { fn serialize(&self, buffer: &mut T) { - if let ElementSegment::ActiveImplicitTableIndex { offset, fn_indices } = &self { - buffer.append_u8(ElementSegmentFormatId::ActiveImplicitTableIndex as u8); - offset.serialize(buffer); - fn_indices.serialize(buffer); - } else { - internal_error!("Unsupported ElementSegment {:?}", self) - } + buffer.append_u8(ElementSegmentFormatId::ActiveImplicitTableIndex as u8); + self.offset.serialize(buffer); + self.fn_indices.serialize(buffer); } } @@ -940,11 +850,7 @@ impl<'a> ElementSection<'a> { pub fn indirect_callees(&self, arena: &'a Bump) -> Vec<'a, u32> { let mut result = bumpalo::vec![in arena]; for segment in self.segments.iter() { - if let ElementSegment::ActiveImplicitTableIndex { fn_indices, .. } = segment { - result.extend_from_slice(fn_indices); - } else { - internal_error!("Unsupported ElementSegment {:?}", self) - } + result.extend_from_slice(&segment.fn_indices); } result } From 8d484e8c9ee956d4582d5dc5953fd6d89a0b668e Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Mon, 28 Feb 2022 00:53:01 +0000 Subject: [PATCH 18/30] wasm: tests read panic message from a global pointer --- compiler/test_gen/src/helpers/wasm.rs | 62 ++++++------------- .../test_gen/src/helpers/wasm_test_platform.c | 3 - 2 files changed, 19 insertions(+), 46 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index 100669c3f7..fb19a8a677 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -1,6 +1,8 @@ use core::cell::Cell; +use libc::c_char; use roc_gen_wasm::wasm_module::{Export, ExportType}; use std::collections::hash_map::DefaultHasher; +use std::ffi::CStr; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; @@ -17,7 +19,7 @@ const OUT_DIR_VAR: &str = "TEST_GEN_OUT"; const TEST_WRAPPER_NAME: &str = "test_wrapper"; const INIT_REFCOUNT_NAME: &str = "init_refcount_test"; -const GET_PANIC_MSG_NAME: &str = "get_panic_msg"; +const PANIC_MSG_NAME: &str = "panic_msg"; fn promote_expr_to_module(src: &str) -> String { let mut buffer = String::from("app \"test\" provides [ main ] to \"./platform\"\n\nmain =\n"); @@ -135,15 +137,6 @@ fn compile_roc_to_wasm_bytes<'a, T: Wasm32Result>( index: init_refcount_idx, }); - // Export the getter function for panic messages - let get_panic_msg_bytes = GET_PANIC_MSG_NAME.as_bytes(); - let get_panic_msg_idx = module.names.functions[get_panic_msg_bytes]; - module.export.append(Export { - name: arena.alloc_slice_copy(get_panic_msg_bytes), - ty: ExportType::Func, - index: get_panic_msg_idx, - }); - module.remove_dead_preloads(env.arena, called_preload_fns); let mut app_module_bytes = std::vec::Vec::with_capacity(module.size()); @@ -202,10 +195,22 @@ where let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap(); match test_wrapper.call(&[]) { - Err(e) => match get_panic_msg(&instance, memory) { - Ok(msg) => Err(msg), - Err(_) => Err(format!("{}", e)), - }, + Err(e) => { + // Check if the error is from roc_panic + // Our test roc_panic stores a pointer to its message in a global variable so we can find it. + let panic_msg_global = instance.exports.get_global(PANIC_MSG_NAME).unwrap(); + let panic_msg_index = panic_msg_global.get().unwrap_i32() as usize; + if panic_msg_index == 0 { + // Pointer is null. The error isn't a roc_panic. + Err(e.to_string()) + } else { + let memory_bytes = unsafe { memory.data_unchecked() }; + let msg_ptr = memory_bytes[panic_msg_index..].as_ptr() as *const c_char; + let msg_cstr = unsafe { CStr::from_ptr(msg_ptr) }; + let msg_str = msg_cstr.to_str().unwrap(); + Err(msg_str.into()) + } + } Ok(result) => { let address = result[0].unwrap_i32(); @@ -226,35 +231,6 @@ where } } -/// Call our test platform's getter function for panic messages -fn get_panic_msg(instance: &wasmer::Instance, memory: &wasmer::Memory) -> Result { - let msg_getter = instance - .exports - .get_function(GET_PANIC_MSG_NAME) - .map_err(|e| format!("{:?}", e))?; - - let msg_result = msg_getter.call(&[]).map_err(|e| format!("{:?}", e))?; - - let msg_addr: i32 = match msg_result[0] { - wasmer::Value::I32(a) => a, - _ => panic!(), - }; - if msg_addr == 0 { - return Err("no panic msg".to_string()); - } - - let msg_index = msg_addr as usize; - let memory_bytes = unsafe { memory.data_unchecked() }; - let msg_len = memory_bytes[msg_index..] - .iter() - .position(|c| *c == 0) - .unwrap(); - - let msg_bytes = &memory_bytes[msg_index..msg_len]; - let msg = unsafe { String::from_utf8_unchecked(msg_bytes.to_vec()) }; - Ok(msg) -} - #[allow(dead_code)] pub fn assert_wasm_refcounts_help( src: &str, diff --git a/compiler/test_gen/src/helpers/wasm_test_platform.c b/compiler/test_gen/src/helpers/wasm_test_platform.c index c67853604e..2a70f8dc65 100644 --- a/compiler/test_gen/src/helpers/wasm_test_platform.c +++ b/compiler/test_gen/src/helpers/wasm_test_platform.c @@ -124,9 +124,6 @@ void roc_dealloc(void *ptr, unsigned int alignment) // Allow the test to probe the panic message char* panic_msg; -char* get_panic_msg() { - return panic_msg; -} void roc_panic(char *msg, unsigned int tag_id) { From 21a3ed4258d528d347b30ea29762a89922f36dc2 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Mon, 28 Feb 2022 21:51:07 +0000 Subject: [PATCH 19/30] wasm: fix dead code elimination in the case where there are live imports --- .../gen_wasm/src/wasm_module/dead_code.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/dead_code.rs b/compiler/gen_wasm/src/wasm_module/dead_code.rs index ecbb59b895..5e52c9a172 100644 --- a/compiler/gen_wasm/src/wasm_module/dead_code.rs +++ b/compiler/gen_wasm/src/wasm_module/dead_code.rs @@ -70,12 +70,17 @@ pub fn parse_preloads_call_graph<'a>( defined_fn_signatures: &[u32], indirect_callees: &[u32], ) -> PreloadsCallGraph<'a> { - let mut call_graph = - PreloadsCallGraph::new(arena, imported_fn_signatures.len(), defined_fn_signatures.len()); + let mut call_graph = PreloadsCallGraph::new( + arena, + imported_fn_signatures.len(), + defined_fn_signatures.len(), + ); // Function type signatures, used for indirect calls - let mut signatures = - Vec::with_capacity_in(imported_fn_signatures.len() + defined_fn_signatures.len(), arena); + let mut signatures = Vec::with_capacity_in( + imported_fn_signatures.len() + defined_fn_signatures.len(), + arena, + ); signatures.extend_from_slice(imported_fn_signatures); signatures.extend_from_slice(defined_fn_signatures); @@ -211,11 +216,13 @@ pub fn copy_preloads_shrinking_dead_fns<'a, T: SerialBuffer>( live_preload_indices.sort_unstable(); live_preload_indices.dedup(); - let mut live_iter = live_preload_indices.iter(); + let mut live_iter = live_preload_indices + .into_iter() + .filter(|f| (*f as usize) >= preload_idx_start); let mut next_live_idx = live_iter.next(); for i in preload_idx_start..call_graph.num_preloads { match next_live_idx { - Some(live) if *live as usize == i => { + Some(live) if live as usize == i => { next_live_idx = live_iter.next(); let live_body_start = call_graph.code_offsets[i] as usize; let live_body_end = call_graph.code_offsets[i + 1] as usize; From 85ba3154ec0e77eddfb41139fa3b0b69406ad3a4 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Mon, 28 Feb 2022 21:51:45 +0000 Subject: [PATCH 20/30] wasm: fix roc_panic message retrieval --- compiler/test_gen/src/helpers/wasm.rs | 42 ++++++++++++++++++--------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index fb19a8a677..1698ad6a0f 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -1,8 +1,6 @@ use core::cell::Cell; -use libc::c_char; use roc_gen_wasm::wasm_module::{Export, ExportType}; use std::collections::hash_map::DefaultHasher; -use std::ffi::CStr; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; @@ -196,19 +194,10 @@ where match test_wrapper.call(&[]) { Err(e) => { - // Check if the error is from roc_panic - // Our test roc_panic stores a pointer to its message in a global variable so we can find it. - let panic_msg_global = instance.exports.get_global(PANIC_MSG_NAME).unwrap(); - let panic_msg_index = panic_msg_global.get().unwrap_i32() as usize; - if panic_msg_index == 0 { - // Pointer is null. The error isn't a roc_panic. - Err(e.to_string()) + if let Some(msg) = get_roc_panic_msg(&instance, memory) { + Err(msg) } else { - let memory_bytes = unsafe { memory.data_unchecked() }; - let msg_ptr = memory_bytes[panic_msg_index..].as_ptr() as *const c_char; - let msg_cstr = unsafe { CStr::from_ptr(msg_ptr) }; - let msg_str = msg_cstr.to_str().unwrap(); - Err(msg_str.into()) + Err(e.to_string()) } } Ok(result) => { @@ -231,6 +220,31 @@ where } } +/// Our test roc_panic stores a pointer to its message in a global variable so we can find it. +fn get_roc_panic_msg(instance: &wasmer::Instance, memory: &Memory) -> Option { + let memory_bytes = unsafe { memory.data_unchecked() }; + + // We need to dereference twice! + // The Wasm Global only points at the memory location of the C global value + let panic_msg_global = instance.exports.get_global(PANIC_MSG_NAME).unwrap(); + let global_addr = panic_msg_global.get().unwrap_i32() as usize; + let global_ptr = memory_bytes[global_addr..].as_ptr() as *const u32; + + // Dereference again to find the bytes of the message string + let msg_addr = unsafe { *global_ptr }; + if msg_addr == 0 { + return None; + } + let msg_index = msg_addr as usize; + let msg_len = memory_bytes[msg_index..] + .iter() + .position(|c| *c == 0) + .unwrap(); + let msg_bytes = memory_bytes[msg_index..][..msg_len].to_vec(); + let msg = unsafe { String::from_utf8_unchecked(msg_bytes) }; + Some(msg) +} + #[allow(dead_code)] pub fn assert_wasm_refcounts_help( src: &str, From 9bf0fdb808bd6b91e640568cfe8a6a4b01798f0e Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Tue, 1 Mar 2022 22:04:29 +0000 Subject: [PATCH 21/30] Fix clippy errors --- compiler/gen_wasm/src/wasm_module/sections.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index 99b445e100..bad1a172e9 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -1205,7 +1205,7 @@ impl<'a> Serialize for NameSection<'a> { impl<'a> Debug for NameSection<'a> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "NameSection\n")?; + writeln!(f, "NameSection")?; // We want to display index->name because it matches the binary format and looks nicer. // But our hashmap is name->index because that's what code gen wants to look up. @@ -1217,7 +1217,7 @@ impl<'a> Debug for NameSection<'a> { for (index, name) in by_index.iter() { let name_str = unsafe { std::str::from_utf8_unchecked(name) }; - write!(f, " {:4}: {}\n", index, name_str)?; + writeln!(f, " {:4}: {}", index, name_str)?; } Ok(()) From 896715cd586856e4170f94b35b1b0b852034d41f Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 2 Mar 2022 08:02:21 +0000 Subject: [PATCH 22/30] wasm: fix Cargo dependency for ARM --- compiler/test_gen/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/test_gen/Cargo.toml b/compiler/test_gen/Cargo.toml index 80348a1dd9..203b669d48 100644 --- a/compiler/test_gen/Cargo.toml +++ b/compiler/test_gen/Cargo.toml @@ -39,7 +39,6 @@ libc = "0.2.106" inkwell = { path = "../../vendor/inkwell" } target-lexicon = "0.12.2" libloading = "0.7.1" -wasmer = { version = "2.0.0", default-features = false, features = ["default-singlepass", "default-universal"] } wasmer-wasi = "2.0.0" tempfile = "3.2.0" indoc = "1.0.3" From 5882ee4af05cba702cb62e01a9d7cc3625e42d24 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 2 Mar 2022 08:03:00 +0000 Subject: [PATCH 23/30] wasm: add comment on why DataSection::bytes is public --- compiler/gen_wasm/src/wasm_module/sections.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index bad1a172e9..a87cfa7865 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -1022,7 +1022,7 @@ impl Serialize for DataSegment<'_> { #[derive(Debug)] pub struct DataSection<'a> { count: u32, - pub bytes: Vec<'a, u8>, + pub bytes: Vec<'a, u8>, // public so backend.rs can calculate addr of first string } impl<'a> DataSection<'a> { From 561a1d16ff46480eca021a02b067b1bc402f1ca9 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 2 Mar 2022 08:05:48 +0000 Subject: [PATCH 24/30] wasm: add expect_runtime_error_panic test macro --- compiler/test_gen/src/helpers/wasm.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index 1698ad6a0f..a09351837f 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -354,12 +354,9 @@ macro_rules! assert_evals_to { $crate::helpers::wasm::assert_evals_to!($src, $expected, $ty, $transform, false); }; - // LLVM tests call the last arg $ignore_problems rather than $expect_panic - // Either way, it's only `true` for tests that use `should_panic`. - // In Wasm, `should_panic` only works if we spawn a child process. - ($src:expr, $expected:expr, $ty:ty, $transform:expr, $expect_panic: expr) => {{ + ($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{ let phantom = std::marker::PhantomData; - let _ = $expect_panic; // TODO + let _ = $ignore_problems; // Always ignore "problems"! One backend (LLVM) is enough to cover them. match $crate::helpers::wasm::assert_evals_to_help::<$ty>($src, phantom) { Err(msg) => panic!("{}", msg), Ok(actual) => { @@ -369,6 +366,19 @@ macro_rules! assert_evals_to { }}; } +#[allow(unused_macros)] +macro_rules! expect_runtime_error_panic { + ($src:expr) => {{ + $crate::helpers::wasm::assert_evals_to!( + $src, + false, // fake value/type for eval + bool, + $crate::helpers::wasm::identity, + true // ignore problems + ); + }}; +} + #[allow(dead_code)] pub fn identity(value: T) -> T { value @@ -396,5 +406,8 @@ macro_rules! assert_refcounts { #[allow(unused_imports)] pub(crate) use assert_evals_to; +#[allow(unused_imports)] +pub(crate) use expect_runtime_error_panic; + #[allow(unused_imports)] pub(crate) use assert_refcounts; From a7899bddedda633741ca9f85916b3701c5224774 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Wed, 2 Mar 2022 08:15:36 +0000 Subject: [PATCH 25/30] wasm: enable another roc_panic test --- compiler/test_gen/src/gen_records.rs | 24 +++++++++++++++++++----- compiler/test_gen/src/gen_tags.rs | 2 +- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/compiler/test_gen/src/gen_records.rs b/compiler/test_gen/src/gen_records.rs index 520db04b03..f2e79a80df 100644 --- a/compiler/test_gen/src/gen_records.rs +++ b/compiler/test_gen/src/gen_records.rs @@ -1,14 +1,11 @@ #[cfg(feature = "gen-llvm")] -use crate::helpers::llvm::assert_evals_to; - -#[cfg(feature = "gen-llvm")] -use crate::helpers::llvm::expect_runtime_error_panic; +use crate::helpers::llvm::{assert_evals_to, expect_runtime_error_panic}; #[cfg(feature = "gen-dev")] use crate::helpers::dev::assert_evals_to; #[cfg(feature = "gen-wasm")] -use crate::helpers::wasm::assert_evals_to; +use crate::helpers::wasm::{assert_evals_to, expect_runtime_error_panic}; // use crate::assert_wasm_evals_to as assert_evals_to; use indoc::indoc; @@ -1044,6 +1041,7 @@ fn different_proc_types_specialized_to_same_layout() { #[cfg(any(feature = "gen-llvm"))] #[should_panic( // TODO: something upstream is escaping the ' + // NOTE: Are we sure it's upstream? It's not escaped in gen-wasm version below! expected = r#"Roc failed with message: "Can\'t create record with improper layout""# )] fn call_with_bad_record_runtime_error() { @@ -1058,3 +1056,19 @@ fn call_with_bad_record_runtime_error() { "# )) } + +#[test] +#[cfg(any(feature = "gen-wasm"))] +#[should_panic(expected = r#"Can't create record with improper layout"#)] +fn call_with_bad_record_runtime_error() { + expect_runtime_error_panic!(indoc!( + r#" + app "test" provides [ main ] to "./platform" + + main = + get : {a: Bool} -> Bool + get = \{a} -> a + get {b: ""} + "# + )) +} diff --git a/compiler/test_gen/src/gen_tags.rs b/compiler/test_gen/src/gen_tags.rs index 91b72bf27f..8288a46925 100644 --- a/compiler/test_gen/src/gen_tags.rs +++ b/compiler/test_gen/src/gen_tags.rs @@ -1222,7 +1222,7 @@ fn applied_tag_function_linked_list() { #[test] #[cfg(any(feature = "gen-llvm", feature = "gen-wasm"))] -#[should_panic(expected = "")] +#[should_panic(expected = "")] // TODO: this only panics because it returns 0 instead of 1! fn tag_must_be_its_own_type() { assert_evals_to!( indoc!( From 20e46fbda4568b29384456d578f9d8318342d6dc Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Fri, 4 Mar 2022 19:12:18 +0000 Subject: [PATCH 26/30] wasm: address PR feedback --- compiler/gen_wasm/src/wasm_module/dead_code.rs | 2 +- compiler/test_gen/src/helpers/wasm.rs | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/compiler/gen_wasm/src/wasm_module/dead_code.rs b/compiler/gen_wasm/src/wasm_module/dead_code.rs index 5e52c9a172..c950484cf1 100644 --- a/compiler/gen_wasm/src/wasm_module/dead_code.rs +++ b/compiler/gen_wasm/src/wasm_module/dead_code.rs @@ -218,7 +218,7 @@ pub fn copy_preloads_shrinking_dead_fns<'a, T: SerialBuffer>( let mut live_iter = live_preload_indices .into_iter() - .filter(|f| (*f as usize) >= preload_idx_start); + .skip_while(|f| (*f as usize) < preload_idx_start); let mut next_live_idx = live_iter.next(); for i in preload_idx_start..call_graph.num_preloads { match next_live_idx { diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index a09351837f..e716c6b266 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -1,6 +1,8 @@ use core::cell::Cell; +use libc::c_char; use roc_gen_wasm::wasm_module::{Export, ExportType}; use std::collections::hash_map::DefaultHasher; +use std::ffi::CStr; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; @@ -236,13 +238,10 @@ fn get_roc_panic_msg(instance: &wasmer::Instance, memory: &Memory) -> Option Date: Fri, 4 Mar 2022 19:51:58 +0000 Subject: [PATCH 27/30] wasm: go back to custom bytes->string transformation CStr version is too hard to get working correctly. I get weird test failures with random characters in the middle of strings. --- compiler/test_gen/src/helpers/wasm.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index e716c6b266..a09351837f 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -1,8 +1,6 @@ use core::cell::Cell; -use libc::c_char; use roc_gen_wasm::wasm_module::{Export, ExportType}; use std::collections::hash_map::DefaultHasher; -use std::ffi::CStr; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::path::{Path, PathBuf}; @@ -238,10 +236,13 @@ fn get_roc_panic_msg(instance: &wasmer::Instance, memory: &Memory) -> Option Date: Sat, 5 Mar 2022 13:37:01 +0000 Subject: [PATCH 28/30] wasm: use a debug-friendly enum for refcount tests --- compiler/test_gen/src/gen_refcount.rs | 102 +++++++++++++------------- compiler/test_gen/src/helpers/mod.rs | 7 ++ compiler/test_gen/src/helpers/wasm.rs | 16 ++-- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/compiler/test_gen/src/gen_refcount.rs b/compiler/test_gen/src/gen_refcount.rs index e16cad765b..25ade5e934 100644 --- a/compiler/test_gen/src/gen_refcount.rs +++ b/compiler/test_gen/src/gen_refcount.rs @@ -1,5 +1,5 @@ #[cfg(feature = "gen-wasm")] -use crate::helpers::wasm::assert_refcounts; +use crate::helpers::{wasm::assert_refcounts, RefCount::*}; #[allow(unused_imports)] use indoc::indoc; @@ -25,8 +25,8 @@ fn str_inc() { ), RocList, &[ - 3, // s - 1 // result + Live(3), // s + Live(1) // result ] ); } @@ -43,7 +43,7 @@ fn str_dealloc() { "# ), bool, - &[0] + &[Deallocated] ); } @@ -60,10 +60,10 @@ fn list_int_inc() { RocList>, &[ // TODO be smarter about coalescing polymorphic list values - 1, // list0 - 1, // list1 - 1, // list2 - 1 // result + Live(1), // list0 + Live(1), // list1 + Live(1), // list2 + Live(1) // result ] ); } @@ -81,10 +81,10 @@ fn list_int_dealloc() { usize, &[ // TODO be smarter about coalescing polymorphic list values - 0, // list0 - 0, // list1 - 0, // list2 - 0 // result + Deallocated, // list0 + Deallocated, // list1 + Deallocated, // list2 + Deallocated // result ] ); } @@ -102,9 +102,9 @@ fn list_str_inc() { ), RocList>, &[ - 6, // s - 2, // list - 1 // result + Live(6), // s + Live(2), // list + Live(1) // result ] ); } @@ -122,9 +122,9 @@ fn list_str_dealloc() { ), usize, &[ - 0, // s - 0, // list - 0 // result + Deallocated, // s + Deallocated, // list + Deallocated // result ] ); } @@ -142,7 +142,7 @@ fn struct_inc() { "# ), [(i64, RocStr, RocStr); 2], - &[4] // s + &[Live(4)] // s ); } @@ -160,7 +160,7 @@ fn struct_dealloc() { "# ), i64, - &[0] // s + &[Deallocated] // s ); } @@ -186,7 +186,7 @@ fn union_nonrecursive_inc() { "# ), (TwoStr, TwoStr, i64), - &[4] + &[Live(4)] ); } @@ -209,7 +209,7 @@ fn union_nonrecursive_dec() { "# ), RocStr, - &[1] // s + &[Live(1)] // s ); } @@ -234,9 +234,9 @@ fn union_recursive_inc() { ), (Pointer, Pointer), &[ - 4, // s - 4, // sym - 2, // e + Live(4), // s + Live(4), // sym + Live(2), // e ] ); } @@ -264,9 +264,9 @@ fn union_recursive_dec() { ), Pointer, &[ - 1, // s - 1, // sym - 0 // e + Live(1), // s + Live(1), // sym + Deallocated // e ] ); } @@ -300,13 +300,13 @@ fn refcount_different_rosetrees_inc() { ), (Pointer, Pointer), &[ - 2, // s - 3, // i1 - 2, // s1 - 1, // [i1, i1] - 1, // i2 - 1, // [s1, s1] - 1 // s2 + Live(2), // s + Live(3), // i1 + Live(2), // s1 + Live(1), // [i1, i1] + Live(1), // i2 + Live(1), // [s1, s1] + Live(1) // s2 ] ); } @@ -341,13 +341,13 @@ fn refcount_different_rosetrees_dec() { ), i64, &[ - 0, // s - 0, // i1 - 0, // s1 - 0, // [i1, i1] - 0, // i2 - 0, // [s1, s1] - 0, // s2 + Deallocated, // s + Deallocated, // i1 + Deallocated, // s1 + Deallocated, // [i1, i1] + Deallocated, // i2 + Deallocated, // [s1, s1] + Deallocated, // s2 ] ); } @@ -370,10 +370,10 @@ fn union_linked_list_inc() { ), (Pointer, Pointer), &[ - 6, // s - 2, // Cons - 2, // Cons - 2, // Cons + Live(6), // s + Live(2), // Cons + Live(2), // Cons + Live(2), // Cons ] ); } @@ -398,10 +398,10 @@ fn union_linked_list_dec() { ), RocStr, &[ - 1, // s - 0, // Cons - 0, // Cons - 0, // Cons + Live(1), // s + Deallocated, // Cons + Deallocated, // Cons + Deallocated, // Cons ] ); } @@ -434,6 +434,6 @@ fn union_linked_list_long_dec() { "# ), i64, - &[0; 1_000] + &[Deallocated; 1_000] ); } diff --git a/compiler/test_gen/src/helpers/mod.rs b/compiler/test_gen/src/helpers/mod.rs index 10e3c70e8b..53b642e3f6 100644 --- a/compiler/test_gen/src/helpers/mod.rs +++ b/compiler/test_gen/src/helpers/mod.rs @@ -55,3 +55,10 @@ where { run_test() } + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RefCount { + Live(u32), + Deallocated, + Constant, +} diff --git a/compiler/test_gen/src/helpers/wasm.rs b/compiler/test_gen/src/helpers/wasm.rs index a09351837f..02a02bb8d7 100644 --- a/compiler/test_gen/src/helpers/wasm.rs +++ b/compiler/test_gen/src/helpers/wasm.rs @@ -6,6 +6,7 @@ use std::marker::PhantomData; use std::path::{Path, PathBuf}; use wasmer::{Memory, WasmPtr}; +use super::RefCount; use crate::helpers::from_wasmer_memory::FromWasmerMemory; use roc_collections::all::{MutMap, MutSet}; use roc_gen_wasm::wasm32_result::Wasm32Result; @@ -250,7 +251,7 @@ pub fn assert_wasm_refcounts_help( src: &str, phantom: PhantomData, num_refcounts: usize, -) -> Result, String> +) -> Result, String> where T: FromWasmerMemory + Wasm32Result, { @@ -296,12 +297,15 @@ where for i in 0..num_refcounts { let rc_ptr = refcount_ptrs[i].get(); let rc = if rc_ptr.offset() == 0 { - // RC pointer has been set to null, which means the value has been freed. - // In tests, we simply represent this as zero refcount. - 0 + RefCount::Deallocated } else { - let rc_encoded = rc_ptr.deref(memory).unwrap().get(); - (rc_encoded - i32::MIN + 1) as u32 + let rc_encoded: i32 = rc_ptr.deref(memory).unwrap().get(); + if rc_encoded == 0 { + RefCount::Constant + } else { + let rc = rc_encoded - i32::MIN + 1; + RefCount::Live(rc as u32) + } }; refcounts.push(rc); } From 9ae26c5aacbf69b7d6fb29251ead66f8c81b2176 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Tue, 8 Mar 2022 08:56:34 +0000 Subject: [PATCH 29/30] wasm: use __data_end to account for all constant data including zero (bss) data --- compiler/gen_wasm/src/backend.rs | 12 +-- compiler/gen_wasm/src/wasm_module/sections.rs | 78 +++++++++++++------ .../gen_wasm/src/wasm_module/serialize.rs | 11 ++- 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/compiler/gen_wasm/src/backend.rs b/compiler/gen_wasm/src/backend.rs index b0901f8c46..f7c4fe232d 100644 --- a/compiler/gen_wasm/src/backend.rs +++ b/compiler/gen_wasm/src/backend.rs @@ -29,11 +29,6 @@ use crate::{ PTR_SIZE, PTR_TYPE, STACK_POINTER_GLOBAL_ID, STACK_POINTER_NAME, TARGET_INFO, }; -/// The memory address where the constants data will be loaded during module instantiation. -/// We avoid address zero and anywhere near it. They're valid addresses but maybe bug-prone. -/// Follow Emscripten's example by leaving 1kB unused (though 4 bytes would probably do!) -const CONST_SEGMENT_BASE_ADDR: u32 = 1024; - pub struct WasmBackend<'a> { pub env: &'a Env<'a>, interns: &'a mut Interns, @@ -57,7 +52,6 @@ pub struct WasmBackend<'a> { } impl<'a> WasmBackend<'a> { - #[allow(clippy::too_many_arguments)] pub fn new( env: &'a Env<'a>, interns: &'a mut Interns, @@ -78,8 +72,10 @@ impl<'a> WasmBackend<'a> { index: STACK_POINTER_GLOBAL_ID, }); - // TODO: Examine the actual address ranges of every preloaded data segment in case they are not simply sequential - let next_constant_addr = CONST_SEGMENT_BASE_ADDR + module.data.bytes.len() as u32; + // The preloaded binary has a global to tell us where its data section ends + // Note: We need this to account for zero data (.bss), which doesn't have an explicit DataSegment! + let data_end_idx = module.export.globals_lookup["__data_end".as_bytes()]; + let next_constant_addr = module.global.parse_u32_at_index(data_end_idx); WasmBackend { env, diff --git a/compiler/gen_wasm/src/wasm_module/sections.rs b/compiler/gen_wasm/src/wasm_module/sections.rs index a87cfa7865..f64db3591b 100644 --- a/compiler/gen_wasm/src/wasm_module/sections.rs +++ b/compiler/gen_wasm/src/wasm_module/sections.rs @@ -12,7 +12,8 @@ use super::dead_code::{ use super::linking::RelocationEntry; use super::opcodes::OpCode; use super::serialize::{ - parse_u32_or_panic, SerialBuffer, Serialize, SkipBytes, MAX_SIZE_ENCODED_U32, + parse_string_bytes, parse_u32_or_panic, SerialBuffer, Serialize, SkipBytes, + MAX_SIZE_ENCODED_U32, }; use super::{CodeBuilder, ValueType}; @@ -565,6 +566,26 @@ pub enum ConstExpr { F64(f64), } +impl ConstExpr { + fn parse_u32(bytes: &[u8], cursor: &mut usize) -> u32 { + let err = || internal_error!("Invalid ConstExpr. Expected i32."); + + if bytes[*cursor] != OpCode::I32CONST as u8 { + err(); + } + *cursor += 1; + + let value = parse_u32_or_panic(bytes, cursor); + + if bytes[*cursor] != OpCode::END as u8 { + err(); + } + *cursor += 1; + + value + } +} + impl Serialize for ConstExpr { fn serialize(&self, buffer: &mut T) { match self { @@ -589,6 +610,15 @@ impl Serialize for ConstExpr { } } +impl SkipBytes for ConstExpr { + fn skip_bytes(bytes: &[u8], cursor: &mut usize) { + while bytes[*cursor] != OpCode::END as u8 { + OpCode::skip_bytes(bytes, cursor); + } + *cursor += 1; + } +} + #[derive(Debug)] pub struct Global { /// Type and mutability of the global @@ -611,16 +641,14 @@ pub struct GlobalSection<'a> { } impl<'a> GlobalSection<'a> { - pub fn new(arena: &'a Bump, globals: &[Global]) -> Self { - let capacity = 13 * globals.len(); - let mut bytes = Vec::with_capacity_in(capacity, arena); - for global in globals { - global.serialize(&mut bytes); - } - GlobalSection { - count: globals.len() as u32, - bytes, + pub fn parse_u32_at_index(&self, index: u32) -> u32 { + let mut cursor = 0; + for _ in 0..index { + GlobalType::skip_bytes(&self.bytes, &mut cursor); + ConstExpr::skip_bytes(&self.bytes, &mut cursor); } + GlobalType::skip_bytes(&self.bytes, &mut cursor); + ConstExpr::parse_u32(&self.bytes, &mut cursor) } pub fn append(&mut self, global: Global) { @@ -666,15 +694,15 @@ pub struct Export<'a> { } impl<'a> Export<'a> { - fn parse_type(bytes: &[u8], cursor: &mut usize) -> ExportType { - String::skip_bytes(bytes, cursor); // name + fn parse(arena: &'a Bump, bytes: &[u8], cursor: &mut usize) -> Self { + let name = parse_string_bytes(arena, bytes, cursor); let ty = ExportType::from(bytes[*cursor]); *cursor += 1; - u32::skip_bytes(bytes, cursor); // index + let index = parse_u32_or_panic(bytes, cursor); - ty + Export { name, ty, index } } } @@ -690,7 +718,10 @@ impl Serialize for Export<'_> { pub struct ExportSection<'a> { pub count: u32, pub bytes: Vec<'a, u8>, + /// List of exported functions to keep during dead-code-elimination pub function_indices: Vec<'a, u32>, + /// name -> index + pub globals_lookup: MutMap<&'a [u8], u32>, } impl<'a> ExportSection<'a> { @@ -713,6 +744,7 @@ impl<'a> ExportSection<'a> { count: 0, bytes: Vec::with_capacity_in(256, arena), function_indices: Vec::with_capacity_in(4, arena), + globals_lookup: MutMap::default(), } } @@ -725,11 +757,14 @@ impl<'a> ExportSection<'a> { let mut body_cursor = 0; for _ in 0..num_exports { let export_start = body_cursor; - let export_type = Export::parse_type(body_bytes, &mut body_cursor); - if matches!(export_type, ExportType::Global) { + let export = Export::parse(arena, body_bytes, &mut body_cursor); + if matches!(export.ty, ExportType::Global) { let global_bytes = &body_bytes[export_start..body_cursor]; export_section.bytes.extend_from_slice(global_bytes); export_section.count += 1; + export_section + .globals_lookup + .insert(export.name, export.index); } } @@ -1146,10 +1181,7 @@ impl<'a> NameSection<'a> { cursor: &mut usize, section_end: usize, ) { - // Custom section name - let section_name_len = parse_u32_or_panic(module_bytes, cursor); - let section_name_end = *cursor + section_name_len as usize; - let section_name = &module_bytes[*cursor..section_name_end]; + let section_name = parse_string_bytes(arena, module_bytes, cursor); if section_name != Self::NAME.as_bytes() { internal_error!( "Expected Custom section {:?}, found {:?}", @@ -1157,7 +1189,6 @@ impl<'a> NameSection<'a> { std::str::from_utf8(section_name) ); } - *cursor = section_name_end; // Find function names subsection let mut found_function_names = false; @@ -1182,10 +1213,7 @@ impl<'a> NameSection<'a> { let num_entries = parse_u32_or_panic(module_bytes, cursor) as usize; for _ in 0..num_entries { let fn_index = parse_u32_or_panic(module_bytes, cursor); - let name_len = parse_u32_or_panic(module_bytes, cursor); - let name_end = *cursor + name_len as usize; - let name_bytes: &[u8] = &module_bytes[*cursor..name_end]; - *cursor = name_end; + let name_bytes = parse_string_bytes(arena, module_bytes, cursor); self.functions .insert(arena.alloc_slice_copy(name_bytes), fn_index); diff --git a/compiler/gen_wasm/src/wasm_module/serialize.rs b/compiler/gen_wasm/src/wasm_module/serialize.rs index 99561309c5..99abd64018 100644 --- a/compiler/gen_wasm/src/wasm_module/serialize.rs +++ b/compiler/gen_wasm/src/wasm_module/serialize.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, iter::FromIterator}; -use bumpalo::collections::vec::Vec; +use bumpalo::{collections::vec::Vec, Bump}; use roc_error_macros::internal_error; /// In the WebAssembly binary format, all integers are variable-length encoded (using LEB-128) @@ -262,6 +262,15 @@ pub fn parse_u32_or_panic(bytes: &[u8], cursor: &mut usize) -> u32 { value } +pub fn parse_string_bytes<'a>(arena: &'a Bump, bytes: &[u8], cursor: &mut usize) -> &'a [u8] { + let len = parse_u32_or_panic(bytes, cursor); + let end = *cursor + len as usize; + let bytes: &[u8] = &bytes[*cursor..end]; + let copy = arena.alloc_slice_copy(bytes); + *cursor = end; + copy +} + /// Skip over serialized bytes for a type /// This may, or may not, require looking at the byte values pub trait SkipBytes { From 7e0b439ef5cc9eeeac892b83ff0df1f6034a1474 Mon Sep 17 00:00:00 2001 From: Brian Carroll Date: Tue, 8 Mar 2022 08:57:52 +0000 Subject: [PATCH 30/30] test_gen: add missing #[test] attribute --- compiler/test_gen/src/gen_records.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/compiler/test_gen/src/gen_records.rs b/compiler/test_gen/src/gen_records.rs index a390fb60c4..0397957edc 100644 --- a/compiler/test_gen/src/gen_records.rs +++ b/compiler/test_gen/src/gen_records.rs @@ -1073,6 +1073,7 @@ fn call_with_bad_record_runtime_error() { )) } +#[test] #[cfg(any(feature = "gen-llvm", feature = "gen-wasm"))] fn generalized_accessor() { assert_evals_to!(