language-servers/roc
1
0
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2025-09-27 05:49:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

moved all crates into seperate folder + related path fixes

Browse source
This commit is contained in:
Anton-4 2022-07-01 17:37:43 +02:00
parent 12ef03bb86
commit eee85fa45d
No known key found for this signature in database
GPG key ID: C954D6E0F9C0ABFD
1063 changed files with 92 additions and 93 deletions
Download patch file Download diff file Expand all files Collapse all files

372
crates/compiler/test_gen/src/helpers/debug-wasm-test.html Normal file
View file

@ -0,0 +1,372 @@
<html>
<head>
<style>
body {
background-color: #111;
color: #ccc;
font-family: sans-serif;
font-size: 18px;
}
section {
max-width: 900px;
margin: 0 auto;
padding: 0 24px;
}
h1 {
margin: 32px auto;
}
li {
margin: 8px;
}
code {
color: #aaa;
background-color: #000;
padding: 1px 4px;
font-size: 16px;
border-radius: 6px;
}
input,
button {
font-size: 20px;
font-weight: bold;
padding: 4px 12px;
}
small {
font-style: italic;
}
#error {
color: #f00;
}
.controls {
margin-top: 64px;
display: flex;
flex-direction: column;
}
.controls button {
margin: 16px;
}
.row {
display: flex;
justify-content: center;
align-items: center;
margin-bottom: 24px;
}
.row-file label {
margin: 0 32px;
}
</style>
</head>
<body>
<section>
<h1>Debug Wasm tests in the browser!</h1>
<p>
You can step through the generated code instruction-by-instruction, and
examine memory contents
</p>
<h3>Steps</h3>
<ul>
<li>
In <code>gen_wasm/src/lib.rs</code>, set
<code>DEBUG_LOG_SETTINGS.keep_test_binary = true</code>
</li>
<li>Run <code>cargo test-gen-wasm -- my_test --nocapture</code></li>
<li>
Look for the path written to the console for
<code>final.wasm</code> and select it in the file picker below
</li>
<li>
Open the browser DevTools <br />
<small> Control+Shift+I or Command+Option+I or F12 </small>
</li>
<li>
Click one of the buttons below, depending on what kind of test it is.
<br />
<small>
Only one of them will work. The other will probably crash or
something.
</small>
</li>
<li>
The debugger should pause just before entering the first Wasm call.
Step into a couple of Wasm calls until you reach your test code in
<code>$#UserApp_main_1</code>
</li>
<li>
Chrome DevTools now has a Memory Inspector panel! In the debugger,
find <code>Module -> memories -> $memory</code>. Right click and
select "Reveal in Memory Inspector"
</li>
</ul>
<div class="controls">
<div class="row row-file">
<label for="wasm-file">Select final.wasm</label>
<input id="wasm-file" type="file" />
</div>
<div id="error" class="row"></div>
<div class="row">
<button id="button-expression">Run as Roc expression test</button>
<button id="button-refcount">Run as reference counting test</button>
</div>
</div>
</section>
<script>
const file_input = document.getElementById("wasm-file");
const button_expression = document.getElementById("button-expression");
const button_refcount = document.getElementById("button-refcount");
const error_box = document.getElementById("error");
button_expression.onclick = runExpressionTest;
button_refcount.onclick = runRefcountTest;
file_input.onchange = function () {
error_box.innerHTML = "";
};
async function runExpressionTest() {
const file = getFile();
const instance = await compileFileToInstance(file);
debugger; // Next call is Wasm! Step into test_wrapper, then $#UserApp_main_1
instance.exports.test_wrapper();
}
async function runRefcountTest() {
const file = getFile();
const instance = await compileFileToInstance(file);
const MAX_ALLOCATIONS = 100;
const refcount_vector_addr =
instance.exports.init_refcount_test(MAX_ALLOCATIONS);
debugger; // Next call is Wasm! Step into test_wrapper, then $#UserApp_main_1
instance.exports.test_wrapper();
const words = new Uint32Array(instance.exports.memory.buffer);
function deref(addr8) {
return words[addr8 >> 2];
}
const actual_len = deref(refcount_vector_addr);
const rc_pointers = [];
for (let i = 0; i < actual_len; i++) {
const offset = (1 + i) << 2;
const rc_ptr = deref(refcount_vector_addr + offset);
rc_pointers.push(rc_ptr);
}
const rc_encoded = rc_pointers.map((ptr) => ptr && deref(ptr));
const rc_encoded_hex = rc_encoded.map((x) =>
x ? x.toString(16) : "(deallocated)"
);
const rc_values = rc_encoded.map((x) => x && x - 0x80000000 + 1);
console.log({ rc_values, rc_encoded_hex });
}
function getFile() {
const { files } = file_input;
if (!files.length) {
const msg = "Select a file!";
error_box.innerHTML = msg;
throw new Error(msg);
}
return files[0];
}
async function compileFileToInstance(file) {
const buffer = await file.arrayBuffer();
const wasiLinkObject = {};
const importObject = createFakeWasiImports(wasiLinkObject);
const result = await WebAssembly.instantiate(buffer, importObject);
wasiLinkObject.memory8 = new Uint8Array(
result.instance.exports.memory.buffer
);
wasiLinkObject.memory32 = new Uint32Array(
result.instance.exports.memory.buffer
);
return result.instance;
}
// If you print to stdout (for example in the platform), it calls these WASI imports.
// This implementation uses console.log
function createFakeWasiImports(wasiLinkObject) {
const decoder = new TextDecoder();
// fd_close : (i32) -> i32
// Close a file descriptor. Note: This is similar to close in POSIX.
// https://docs.rs/wasi/latest/wasi/wasi_snapshot_preview1/fn.fd_close.html
function fd_close(fd) {
console.warn(`fd_close: ${{ fd }}`);
return 0; // error code
}
// fd_fdstat_get : (i32, i32) -> i32
// Get the attributes of a file descriptor.
// https://docs.rs/wasi/latest/wasi/wasi_snapshot_preview1/fn.fd_fdstat_get.html
function fd_fdstat_get(fd, stat_mut_ptr) {
/*
Tell WASI that stdout is a tty (no seek or tell)
https://github.com/WebAssembly/wasi-libc/blob/659ff414560721b1660a19685110e484a081c3d4/libc-bottom-half/sources/isatty.c
*Not* a tty if:
(statbuf.fs_filetype != __WASI_FILETYPE_CHARACTER_DEVICE ||
(statbuf.fs_rights_base & (__WASI_RIGHTS_FD_SEEK | __WASI_RIGHTS_FD_TELL)) != 0)
So it's sufficient to set:
.fs_filetype = __WASI_FILETYPE_CHARACTER_DEVICE
.fs_rights_base = 0
https://github.com/WebAssembly/wasi-libc/blob/659ff414560721b1660a19685110e484a081c3d4/libc-bottom-half/headers/public/wasi/api.h
typedef uint8_t __wasi_filetype_t;
typedef uint16_t __wasi_fdflags_t;
typedef uint64_t __wasi_rights_t;
#define __WASI_FILETYPE_CHARACTER_DEVICE (UINT8_C(2))
typedef struct __wasi_fdstat_t { // 24 bytes total
__wasi_filetype_t fs_filetype; // 1 byte
// 1 byte padding
__wasi_fdflags_t fs_flags; // 2 bytes
// 4 bytes padding
__wasi_rights_t fs_rights_base; // 8 bytes
__wasi_rights_t fs_rights_inheriting; // 8 bytes
} __wasi_fdstat_t;
*/
// console.warn(`fd_fdstat_get: ${{ fd, stat_mut_ptr }}`);
const WASI_FILETYPE_CHARACTER_DEVICE = 2;
wasiLinkObject.memory8[stat_mut_ptr] = WASI_FILETYPE_CHARACTER_DEVICE;
wasiLinkObject.memory8
.slice(stat_mut_ptr + 1, stat_mut_ptr + 24)
.fill(0);
return 0; // error code
}
// fd_seek : (i32, i64, i32, i32) -> i32
// Move the offset of a file descriptor. Note: This is similar to lseek in POSIX.
// https://docs.rs/wasi/latest/wasi/wasi_snapshot_preview1/fn.fd_seek.html
function fd_seek(fd, offset, whence, newoffset_mut_ptr) {
console.warn(`fd_seek: ${{ fd, offset, whence, newoffset_mut_ptr }}`);
return 0;
}
// fd_write : (i32, i32, i32, i32) -> i32
// Write to a file descriptor. Note: This is similar to `writev` in POSIX.
// https://docs.rs/wasi/latest/wasi/wasi_snapshot_preview1/fn.fd_write.html
function fd_write(fd, iovs_ptr, iovs_len, nwritten_mut_ptr) {
let string_buffer = "";
let nwritten = 0;
const STDOUT = 1;
for (let i = 0; i < iovs_len; i++) {
const index32 = iovs_ptr >> 2;
const base = wasiLinkObject.memory32[index32];
const len = wasiLinkObject.memory32[index32 + 1];
iovs_ptr += 8;
if (!len) continue;
nwritten += len;
// For some reason we often get negative-looking buffer lengths with junk data.
// Just skip the console.log, but still increase nwritten or it will loop forever.
// Dunno why this happens, but it's working fine for printf debugging ¯\_(ツ)_/¯
if (len >> 31) {
break;
}
const buf = wasiLinkObject.memory8.slice(base, base + len);
const chunk = decoder.decode(buf);
string_buffer += chunk;
}
wasiLinkObject.memory32[nwritten_mut_ptr >> 2] = nwritten;
if (string_buffer) {
if (fd === STDOUT) {
console.log(string_buffer);
} else {
console.error(string_buffer);
}
}
return 0;
}
// proc_exit : (i32) -> nil
function proc_exit(exit_code) {
throw new Error(`Wasm exited with code ${exit_code}`);
}
// Signatures from wasm_test_platform.o
const sig2 = (i32) => {};
const sig6 = (i32a, i32b) => 0;
const sig7 = (i32a, i32b, i32c) => 0;
const sig9 = (i32a, i64b, i32c) => 0;
const sig10 = (i32a, i64b, i64c, i32d) => 0;
const sig11 = (i32a, i64b, i64c) => 0;
const sig12 = (i32a) => 0;
const sig13 = (i32a, i64b) => 0;
const sig14 = (i32a, i32b, i32c, i64d, i32e) => 0;
const sig15 = (i32a, i32b, i32c, i32d) => 0;
const sig16 = (i32a, i64b, i32c, i32d) => 0;
const sig17 = (i32a, i32b, i32c, i32d, i32e) => 0;
const sig18 = (i32a, i32b, i32c, i32d, i64e, i64f, i32g) => 0;
const sig19 = (i32a, i32b, i32c, i32d, i32e, i32f, i32g) => 0;
const sig20 = (i32a, i32b, i32c, i32d, i32e, i64f, i64g, i32h, i32i) =>
0;
const sig21 = (i32a, i32b, i32c, i32d, i32e, i32f) => 0;
const sig22 = () => 0;
return {
wasi_snapshot_preview1: {
args_get: sig6,
args_sizes_get: sig6,
environ_get: sig6,
environ_sizes_get: sig6,
clock_res_get: sig6,
clock_time_get: sig9,
fd_advise: sig10,
fd_allocate: sig11,
fd_close,
fd_datasync: sig12,
fd_fdstat_get,
fd_fdstat_set_flags: sig6,
fd_fdstat_set_rights: sig11,
fd_filestat_get: sig6,
fd_filestat_set_size: sig13,
fd_filestat_set_times: sig10,
fd_pread: sig14,
fd_prestat_get: sig6,
fd_prestat_dir_name: sig7,
fd_pwrite: sig14,
fd_read: sig15,
fd_readdir: sig14,
fd_renumber: sig6,
fd_seek,
fd_sync: sig12,
fd_tell: sig6,
fd_write,
path_create_directory: sig7,
path_filestat_get: sig17,
path_filestat_set_times: sig18,
path_link: sig19,
path_open: sig20,
path_readlink: sig21,
path_remove_directory: sig7,
path_rename: sig21,
path_symlink: sig17,
path_unlink_file: sig7,
poll_oneoff: sig15,
proc_exit,
proc_raise: sig12,
sched_yield: sig22,
random_get: sig6,
sock_recv: sig21,
sock_send: sig17,
sock_shutdown: sig6,
},
};
}
</script>
</body>
</html>

265
crates/compiler/test_gen/src/helpers/dev.rs Normal file
View file

@ -0,0 +1,265 @@
use libloading::Library;
use roc_build::link::{link, LinkType};
use roc_builtins::bitcode;
use roc_collections::all::MutMap;
use roc_load::Threading;
use roc_region::all::LineInfo;
use tempfile::tempdir;
#[allow(unused_imports)]
use roc_mono::ir::pretty_print_ir_symbols;
#[allow(dead_code)]
fn promote_expr_to_module(src: &str) -> String {
let mut buffer = String::from("app \"test\" provides [main] to \"./platform\"\n\nmain =\n");
for line in src.lines() {
// indent the body!
buffer.push_str(" ");
buffer.push_str(line);
buffer.push('\n');
}
buffer
}
#[allow(dead_code)]
pub fn helper(
arena: &bumpalo::Bump,
src: &str,
_leak: bool,
lazy_literals: bool,
) -> (String, Vec<roc_problem::can::Problem>, Library) {
use std::path::{Path, PathBuf};
let dir = tempdir().unwrap();
let filename = PathBuf::from("Test.roc");
let src_dir = Path::new("fake/test/path");
let app_o_file = dir.path().join("app.o");
let module_src;
let temp;
if src.starts_with("app") {
// this is already a module
module_src = src;
} else {
// this is an expression, promote it to a module
temp = promote_expr_to_module(src);
module_src = &temp;
}
let loaded = roc_load::load_and_monomorphize_from_str(
arena,
filename,
module_src,
src_dir,
Default::default(),
roc_target::TargetInfo::default_x86_64(),
roc_reporting::report::RenderTarget::ColorTerminal,
Threading::Single,
);
let mut loaded = loaded.expect("failed to load module");
use roc_load::MonomorphizedModule;
let MonomorphizedModule {
module_id,
procedures,
mut interns,
exposed_to_host,
..
} = loaded;
// You can comment and uncomment this block out to get more useful information
// while you're working on the dev backend!
{
// println!("=========== Procedures ==========");
// if pretty_print_ir_symbols() {
// println!("");
// for proc in procedures.values() {
// println!("{}", proc.to_pretty(200));
// }
// } else {
// println!("{:?}", procedures.values());
// }
// println!("=================================\n");
// println!("=========== Interns ==========");
// println!("{:?}", interns);
// println!("=================================\n");
// println!("=========== Exposed ==========");
// println!("{:?}", exposed_to_host);
// println!("=================================\n");
}
debug_assert_eq!(exposed_to_host.values.len(), 1);
let main_fn_symbol = loaded.entry_point.symbol;
let main_fn_layout = loaded.entry_point.layout;
let mut layout_ids = roc_mono::layout::LayoutIds::default();
let main_fn_name = layout_ids
.get_toplevel(main_fn_symbol, &main_fn_layout)
.to_exposed_symbol_string(main_fn_symbol, &interns);
let mut lines = Vec::new();
// errors whose reporting we delay (so we can see that code gen generates runtime errors)
let mut delayed_errors = Vec::new();
for (home, (module_path, src)) in loaded.sources {
use roc_reporting::report::{can_problem, type_problem, RocDocAllocator, DEFAULT_PALETTE};
let can_problems = loaded.can_problems.remove(&home).unwrap_or_default();
let type_problems = loaded.type_problems.remove(&home).unwrap_or_default();
let error_count = can_problems.len() + type_problems.len();
if error_count == 0 {
continue;
}
let line_info = LineInfo::new(&src);
let src_lines: Vec<&str> = src.split('\n').collect();
let palette = DEFAULT_PALETTE;
// Report parsing and canonicalization problems
let alloc = RocDocAllocator::new(&src_lines, home, &interns);
use roc_problem::can::Problem::*;
for problem in can_problems.into_iter() {
// Ignore "unused" problems
match problem {
UnusedDef(_, _) | UnusedArgument(_, _, _) | UnusedImport(_, _) => {
delayed_errors.push(problem);
continue;
}
_ => {
let report = can_problem(&alloc, &line_info, module_path.clone(), problem);
let mut buf = String::new();
report.render_color_terminal(&mut buf, &alloc, &palette);
lines.push(buf);
}
}
}
for problem in type_problems {
if let Some(report) = type_problem(&alloc, &line_info, module_path.clone(), problem) {
let mut buf = String::new();
report.render_color_terminal(&mut buf, &alloc, &palette);
lines.push(buf);
}
}
}
if !lines.is_empty() {
println!("{}", lines.join("\n"));
assert_eq!(0, 1, "Mistakes were made");
}
let env = roc_gen_dev::Env {
arena,
module_id,
exposed_to_host: exposed_to_host.values.keys().copied().collect(),
lazy_literals,
generate_allocators: true, // Needed for testing, since we don't have a platform
};
let target = target_lexicon::Triple::host();
let module_object = roc_gen_dev::build_module(&env, &mut interns, &target, procedures);
let module_out = module_object
.write()
.expect("failed to build output object");
std::fs::write(&app_o_file, module_out).expect("failed to write object to file");
// std::fs::copy(&app_o_file, "/tmp/app.o").unwrap();
let (mut child, dylib_path) = link(
&target,
app_o_file.clone(),
// Long term we probably want a smarter way to link in zig builtins.
// With the current method all methods are kept and it adds about 100k to all outputs.
&[
app_o_file.to_str().unwrap(),
bitcode::BUILTINS_HOST_OBJ_PATH,
],
LinkType::Dylib,
)
.expect("failed to link dynamic library");
child.wait().unwrap();
// Load the dylib
let path = dylib_path.as_path().to_str().unwrap();
// std::fs::copy(&path, "/tmp/libapp.so").unwrap();
let lib = unsafe { Library::new(path) }.expect("failed to load shared library");
(main_fn_name, delayed_errors, lib)
}
#[allow(unused_macros)]
macro_rules! assert_evals_to {
($src:expr, $expected:expr, $ty:ty) => {{
assert_evals_to!($src, $expected, $ty, (|val| val));
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
// Same as above, except with an additional transformation argument.
{
assert_evals_to!($src, $expected, $ty, $transform, true);
}
};
($src:expr, $expected:expr, $ty:ty, $transform:expr, $leak:expr) => {
// Run both with and without lazy literal optimization.
{
assert_evals_to!($src, $expected, $ty, $transform, $leak, false);
}
{
assert_evals_to!($src, $expected, $ty, $transform, $leak, true);
}
};
($src:expr, $expected:expr, $ty:ty, $transform:expr, $leak:expr, $lazy_literals:expr) => {
use bumpalo::Bump;
use roc_gen_dev::run_jit_function_raw;
let arena = Bump::new();
let (main_fn_name, errors, lib) =
$crate::helpers::dev::helper(&arena, $src, $leak, $lazy_literals);
let transform = |success| {
let expected = $expected;
let given = $transform(success);
assert_eq!(&given, &expected);
};
run_jit_function_raw!(lib, main_fn_name, $ty, transform, errors)
};
}
#[allow(unused_macros)]
macro_rules! assert_expect_failed {
($src:expr, $expected:expr, $ty:ty, $failures:expr) => {{
use bumpalo::Bump;
use roc_gen_dev::run_jit_function_raw;
let stdlib = roc_builtins::std::standard_stdlib();
let arena = Bump::new();
let (main_fn_name, errors, lib) =
$crate::helpers::dev::helper(&arena, $src, stdlib, true, true);
let transform = |success| {
let expected = $expected;
assert_eq!(&success, &expected);
};
run_jit_function_raw!(lib, main_fn_name, $ty, transform, errors);
}};
}
#[allow(unused_imports)]
pub(crate) use assert_evals_to;
#[allow(unused_imports)]
pub(crate) use assert_expect_failed;

169
crates/compiler/test_gen/src/helpers/from_wasmer_memory.rs Normal file
View file

@ -0,0 +1,169 @@
use roc_gen_wasm::wasm32_sized::Wasm32Sized;
use roc_std::{RocDec, RocList, RocOrder, RocStr};
use std::convert::TryInto;
pub trait FromWasmerMemory: Wasm32Sized {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self;
}
macro_rules! from_wasm_memory_primitive_decode {
($type_name:ident) => {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
use core::mem::MaybeUninit;
let mut output: MaybeUninit<Self> = MaybeUninit::uninit();
let width = std::mem::size_of::<Self>();
let ptr = output.as_mut_ptr();
let raw_ptr = ptr as *mut u8;
let slice = unsafe { std::slice::from_raw_parts_mut(raw_ptr, width) };
let memory_bytes: &[u8] = unsafe { memory.data_unchecked() };
let index = offset as usize;
let wasm_slice = &memory_bytes[index..][..width];
slice.copy_from_slice(wasm_slice);
unsafe { output.assume_init() }
}
};
}
macro_rules! from_wasm_memory_primitive {
($($type_name:ident ,)+) => {
$(
impl FromWasmerMemory for $type_name {
from_wasm_memory_primitive_decode!($type_name);
}
)*
}
}
from_wasm_memory_primitive!(
u8, i8, u16, i16, u32, i32, u64, i64, u128, i128, f32, f64, bool, RocDec, RocOrder,
);
impl FromWasmerMemory for () {
fn decode(_: &wasmer::Memory, _: u32) -> Self {}
}
impl FromWasmerMemory for RocStr {
fn decode(memory: &wasmer::Memory, addr: u32) -> Self {
let memory_bytes = unsafe { memory.data_unchecked() };
let index = addr as usize;
let mut str_bytes = [0; 12];
str_bytes.copy_from_slice(&memory_bytes[index..][..12]);
let str_words: &[u32; 3] = unsafe { std::mem::transmute(&str_bytes) };
let big_elem_ptr = str_words[0] as usize;
let big_length = str_words[1] as usize;
let last_byte = str_bytes[11];
let is_small_str = last_byte >= 0x80;
let slice = if is_small_str {
let small_length = (last_byte & 0x7f) as usize;
&str_bytes[0..small_length]
} else {
&memory_bytes[big_elem_ptr..][..big_length]
};
unsafe { RocStr::from_slice_unchecked(slice) }
}
}
impl<T: FromWasmerMemory + Clone> FromWasmerMemory for RocList<T> {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
let bytes = <u64 as FromWasmerMemory>::decode(memory, offset);
let length = (bytes >> 32) as u32;
let elements = bytes as u32;
let mut items = Vec::with_capacity(length as usize);
for i in 0..length {
let item = <T as FromWasmerMemory>::decode(
memory,
elements + i * <T as Wasm32Sized>::SIZE_OF_WASM as u32,
);
items.push(item);
}
RocList::from_slice(&items)
}
}
impl<T: FromWasmerMemory> FromWasmerMemory for &'_ T {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
let elements = <u32 as FromWasmerMemory>::decode(memory, offset);
let actual = <T as FromWasmerMemory>::decode(memory, elements);
let b = Box::new(actual);
std::boxed::Box::<T>::leak(b)
}
}
impl<T: FromWasmerMemory + Clone, const N: usize> FromWasmerMemory for [T; N] {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
let memory_bytes: &[u8] = unsafe { memory.data_unchecked() };
let index = offset as usize;
debug_assert!(memory_bytes.len() >= index + (N * <T as Wasm32Sized>::SIZE_OF_WASM));
let slice_bytes: &[u8] = &memory_bytes[index..][..N];
let slice: &[T] = unsafe { std::mem::transmute(slice_bytes) };
let array: &[T; N] = slice.try_into().expect("incorrect length");
array.clone()
}
}
impl FromWasmerMemory for usize {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
<u32 as FromWasmerMemory>::decode(memory, offset) as usize
}
}
impl<T: FromWasmerMemory, U: FromWasmerMemory> FromWasmerMemory for (T, U) {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
debug_assert!(
T::ALIGN_OF_WASM >= U::ALIGN_OF_WASM,
"this function does not handle alignment"
);
let t = <T as FromWasmerMemory>::decode(memory, offset);
let u = <U as FromWasmerMemory>::decode(memory, offset + T::ACTUAL_WIDTH as u32);
(t, u)
}
}
impl<T: FromWasmerMemory, U: FromWasmerMemory, V: FromWasmerMemory> FromWasmerMemory for (T, U, V) {
fn decode(memory: &wasmer::Memory, offset: u32) -> Self {
debug_assert!(
T::ALIGN_OF_WASM >= U::ALIGN_OF_WASM,
"this function does not handle alignment"
);
debug_assert!(
U::ALIGN_OF_WASM >= V::ALIGN_OF_WASM,
"this function does not handle alignment"
);
let t = <T as FromWasmerMemory>::decode(memory, offset);
let u = <U as FromWasmerMemory>::decode(memory, offset + T::ACTUAL_WIDTH as u32);
let v = <V as FromWasmerMemory>::decode(
memory,
offset + T::ACTUAL_WIDTH as u32 + U::ACTUAL_WIDTH as u32,
);
(t, u, v)
}
}

673
crates/compiler/test_gen/src/helpers/llvm.rs Normal file
View file

@ -0,0 +1,673 @@
use crate::helpers::from_wasmer_memory::FromWasmerMemory;
use inkwell::module::Module;
use libloading::Library;
use roc_build::link::module_to_dylib;
use roc_build::program::FunctionIterator;
use roc_collections::all::MutSet;
use roc_gen_llvm::llvm::externs::add_default_roc_externs;
use roc_load::Threading;
use roc_mono::ir::OptLevel;
use roc_region::all::LineInfo;
use roc_reporting::report::RenderTarget;
use target_lexicon::Triple;
fn promote_expr_to_module(src: &str) -> String {
let mut buffer = String::from("app \"test\" provides [main] to \"./platform\"\n\nmain =\n");
for line in src.lines() {
// indent the body!
buffer.push_str(" ");
buffer.push_str(line);
buffer.push('\n');
}
buffer
}
#[allow(clippy::too_many_arguments)]
fn create_llvm_module<'a>(
arena: &'a bumpalo::Bump,
src: &str,
is_gen_test: bool,
ignore_problems: bool,
context: &'a inkwell::context::Context,
target: &Triple,
opt_level: OptLevel,
) -> (&'static str, String, &'a Module<'a>) {
use std::path::{Path, PathBuf};
let target_info = roc_target::TargetInfo::from(target);
let filename = PathBuf::from("Test.roc");
let src_dir = Path::new("fake/test/path");
let module_src;
let temp;
if src.starts_with("app") {
// this is already a module
module_src = src;
} else {
// this is an expression, promote it to a module
temp = promote_expr_to_module(src);
module_src = &temp;
}
let loaded = roc_load::load_and_monomorphize_from_str(
arena,
filename,
module_src,
src_dir,
Default::default(),
target_info,
RenderTarget::ColorTerminal,
Threading::Single,
);
let mut loaded = match loaded {
Ok(x) => x,
Err(roc_load::LoadingProblem::FormattedReport(report)) => {
println!("{}", report);
panic!();
}
Err(e) => panic!("{:?}", e),
};
use roc_load::MonomorphizedModule;
let MonomorphizedModule {
procedures,
entry_point,
interns,
..
} = loaded;
let mut lines = Vec::new();
// errors whose reporting we delay (so we can see that code gen generates runtime errors)
let mut delayed_errors = Vec::new();
for (home, (module_path, src)) in loaded.sources {
use roc_reporting::report::{can_problem, type_problem, RocDocAllocator, DEFAULT_PALETTE};
let can_problems = loaded.can_problems.remove(&home).unwrap_or_default();
let type_problems = loaded.type_problems.remove(&home).unwrap_or_default();
let error_count = can_problems.len() + type_problems.len();
if error_count == 0 {
continue;
}
let line_info = LineInfo::new(&src);
let src_lines: Vec<&str> = src.split('\n').collect();
let palette = DEFAULT_PALETTE;
// Report parsing and canonicalization problems
let alloc = RocDocAllocator::new(&src_lines, home, &interns);
use roc_problem::can::Problem::*;
for problem in can_problems.into_iter() {
match problem {
// Ignore "unused" problems
UnusedDef(_, _)
| UnusedArgument(_, _, _)
| UnusedImport(_, _)
| RuntimeError(_)
| UnsupportedPattern(_, _)
| ExposedButNotDefined(_) => {
let report = can_problem(&alloc, &line_info, module_path.clone(), problem);
let mut buf = String::new();
report.render_color_terminal(&mut buf, &alloc, &palette);
delayed_errors.push(buf.clone());
lines.push(buf);
}
// We should be able to compile even when abilities are used as types
AbilityUsedAsType(..) => {}
_ => {
let report = can_problem(&alloc, &line_info, module_path.clone(), problem);
let mut buf = String::new();
report.render_color_terminal(&mut buf, &alloc, &palette);
lines.push(buf);
}
}
}
for problem in type_problems {
if let Some(report) = type_problem(&alloc, &line_info, module_path.clone(), problem) {
let mut buf = String::new();
report.render_color_terminal(&mut buf, &alloc, &palette);
lines.push(buf);
}
}
}
if !lines.is_empty() {
println!("{}", lines.join("\n"));
// only crash at this point if there were no delayed_errors
if delayed_errors.is_empty() && !ignore_problems {
assert_eq!(0, 1, "Mistakes were made");
}
}
let builder = context.create_builder();
let module = roc_gen_llvm::llvm::build::module_from_builtins(target, context, "app");
let module = arena.alloc(module);
let (module_pass, function_pass) =
roc_gen_llvm::llvm::build::construct_optimization_passes(module, opt_level);
let (dibuilder, compile_unit) = roc_gen_llvm::llvm::build::Env::new_debug_info(module);
// mark our zig-defined builtins as internal
use inkwell::attributes::{Attribute, AttributeLoc};
use inkwell::module::Linkage;
let kind_id = Attribute::get_named_enum_kind_id("alwaysinline");
debug_assert!(kind_id > 0);
let attr = context.create_enum_attribute(kind_id, 1);
for function in FunctionIterator::from_module(module) {
let name = function.get_name().to_str().unwrap();
if name.starts_with("roc_builtins") {
if name.starts_with("roc_builtins.expect") {
function.set_linkage(Linkage::External);
} else {
function.set_linkage(Linkage::Internal);
}
}
if name.starts_with("roc_builtins.dict") {
function.add_attribute(AttributeLoc::Function, attr);
}
if name.starts_with("roc_builtins.list") {
function.add_attribute(AttributeLoc::Function, attr);
}
}
// Compile and add all the Procs before adding main
let env = roc_gen_llvm::llvm::build::Env {
arena,
builder: &builder,
dibuilder: &dibuilder,
compile_unit: &compile_unit,
context,
interns,
module,
target_info,
is_gen_test,
// important! we don't want any procedures to get the C calling convention
exposed_to_host: MutSet::default(),
};
// strip Zig debug stuff
module.strip_debug_info();
// Add roc_alloc, roc_realloc, and roc_dealloc, since the repl has no
// platform to provide them.
add_default_roc_externs(&env);
let (main_fn_name, main_fn) = roc_gen_llvm::llvm::build::build_procedures_return_main(
&env,
opt_level,
procedures,
entry_point,
);
env.dibuilder.finalize();
// strip all debug info: we don't use it at the moment and causes weird validation issues
module.strip_debug_info();
// Uncomment this to see the module's un-optimized LLVM instruction output:
// env.module.print_to_stderr();
if main_fn.verify(true) {
function_pass.run_on(&main_fn);
} else {
panic!("Main function {} failed LLVM verification in NON-OPTIMIZED build. Uncomment things nearby to see more details.", main_fn_name);
}
module_pass.run_on(env.module);
// Verify the module
if let Err(errors) = env.module.verify() {
panic!("Errors defining module:\n\n{}", errors.to_string());
}
// Uncomment this to see the module's optimized LLVM instruction output:
// env.module.print_to_stderr();
(main_fn_name, delayed_errors.join("\n"), env.module)
}
#[allow(dead_code)]
#[inline(never)]
pub fn helper<'a>(
arena: &'a bumpalo::Bump,
src: &str,
is_gen_test: bool,
ignore_problems: bool,
context: &'a inkwell::context::Context,
) -> (&'static str, String, Library) {
let target = target_lexicon::Triple::host();
let opt_level = if cfg!(debug_assertions) {
OptLevel::Normal
} else {
OptLevel::Optimize
};
let (main_fn_name, delayed_errors, module) = create_llvm_module(
arena,
src,
is_gen_test,
ignore_problems,
context,
&target,
opt_level,
);
let lib =
module_to_dylib(module, &target, opt_level).expect("Error loading compiled dylib for test");
(main_fn_name, delayed_errors, lib)
}
fn wasm32_target_tripple() -> Triple {
use target_lexicon::{Architecture, BinaryFormat};
let mut triple = Triple::unknown();
triple.architecture = Architecture::Wasm32;
triple.binary_format = BinaryFormat::Wasm;
triple
}
#[allow(dead_code)]
pub fn helper_wasm<'a>(
arena: &'a bumpalo::Bump,
src: &str,
_is_gen_test: bool,
ignore_problems: bool,
context: &'a inkwell::context::Context,
) -> wasmer::Instance {
let target = wasm32_target_tripple();
let opt_level = if cfg!(debug_assertions) {
OptLevel::Normal
} else {
OptLevel::Optimize
};
let is_gen_test = false;
let (_main_fn_name, _delayed_errors, llvm_module) = create_llvm_module(
arena,
src,
is_gen_test,
ignore_problems,
context,
&target,
opt_level,
);
use inkwell::targets::{InitializationConfig, Target, TargetTriple};
let dir = tempfile::tempdir().unwrap();
let dir_path = dir.path();
// let zig_global_cache_path = std::path::PathBuf::from("/home/folkertdev/roc/wasm/mess");
let test_a_path = dir_path.join("test.a");
let test_wasm_path = dir_path.join("libmain.wasm");
Target::initialize_webassembly(&InitializationConfig::default());
let triple = TargetTriple::create("wasm32-unknown-unknown-wasm");
llvm_module.set_triple(&triple);
llvm_module.set_source_file_name("Test.roc");
let target_machine = Target::from_name("wasm32")
.unwrap()
.create_target_machine(
&triple,
"",
"", // TODO: this probably should be TargetMachine::get_host_cpu_features() to enable all features.
inkwell::OptimizationLevel::None,
inkwell::targets::RelocMode::Default,
inkwell::targets::CodeModel::Default,
)
.unwrap();
let file_type = inkwell::targets::FileType::Object;
target_machine
.write_to_file(llvm_module, file_type, &test_a_path)
.unwrap();
use std::process::Command;
Command::new(&crate::helpers::zig_executable())
.current_dir(dir_path)
.args(&[
"wasm-ld",
"/home/folkertdev/roc/wasm/libmain.a",
"/home/folkertdev/roc/wasm/libc.a",
test_a_path.to_str().unwrap(),
"-o",
test_wasm_path.to_str().unwrap(),
"--export-dynamic",
"--allow-undefined",
"--no-entry",
])
.status()
.unwrap();
// now, do wasmer stuff
use wasmer::{Function, Instance, Module, Store};
let store = Store::default();
let module = Module::from_file(&store, &test_wasm_path).unwrap();
// First, we create the `WasiEnv`
use wasmer_wasi::WasiState;
let mut wasi_env = WasiState::new("hello")
// .args(&["world"])
// .env("KEY", "Value")
.finalize()
.unwrap();
// Then, we get the import object related to our WASI
// and attach it to the Wasm instance.
let mut import_object = wasi_env
.import_object(&module)
.unwrap_or_else(|_| wasmer::imports!());
{
let mut exts = wasmer::Exports::new();
let main_function = Function::new_native(&store, fake_wasm_main_function);
let ext = wasmer::Extern::Function(main_function);
exts.insert("main", ext);
let main_function = Function::new_native(&store, wasm_roc_panic);
let ext = wasmer::Extern::Function(main_function);
exts.insert("roc_panic", ext);
import_object.register("env", exts);
}
Instance::new(&module, &import_object).unwrap()
}
#[allow(dead_code)]
fn wasm_roc_panic(address: u32, tag_id: u32) {
match tag_id {
0 => {
let mut string = "";
MEMORY.with(|f| {
let memory = f.borrow().unwrap();
let memory_bytes: &[u8] = unsafe { memory.data_unchecked() };
let index = address as usize;
let slice = &memory_bytes[index..];
let c_ptr: *const u8 = slice.as_ptr();
use std::ffi::CStr;
use std::os::raw::c_char;
let slice = unsafe { CStr::from_ptr(c_ptr as *const c_char) };
string = slice.to_str().unwrap();
});
panic!("Roc failed with message: {:?}", string)
}
_ => todo!(),
}
}
use std::cell::RefCell;
thread_local! {
pub static MEMORY: RefCell<Option<&'static wasmer::Memory>> = RefCell::new(None);
}
#[allow(dead_code)]
fn fake_wasm_main_function(_: u32, _: u32) -> u32 {
panic!("wasm entered the main function; this should never happen!")
}
#[allow(dead_code)]
pub fn assert_wasm_evals_to_help<T>(src: &str, ignore_problems: bool) -> Result<T, String>
where
T: FromWasmerMemory,
{
let arena = bumpalo::Bump::new();
let context = inkwell::context::Context::create();
let is_gen_test = true;
let instance =
crate::helpers::llvm::helper_wasm(&arena, src, is_gen_test, ignore_problems, &context);
let memory = instance.exports.get_memory("memory").unwrap();
crate::helpers::llvm::MEMORY.with(|f| {
*f.borrow_mut() = Some(unsafe { std::mem::transmute(memory) });
});
let test_wrapper = instance.exports.get_function("test_wrapper").unwrap();
match test_wrapper.call(&[]) {
Err(e) => Err(format!("call to `test_wrapper`: {:?}", e)),
Ok(result) => {
let address = result[0].unwrap_i32();
let output = <T as crate::helpers::llvm::FromWasmerMemory>::decode(
memory,
// skip the RocCallResult tag id
address as u32 + 8,
);
Ok(output)
}
}
}
#[allow(unused_macros)]
macro_rules! assert_wasm_evals_to {
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => {
match $crate::helpers::llvm::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems) {
Err(msg) => panic!("Wasm test failed: {:?}", msg),
Ok(actual) => {
#[allow(clippy::bool_assert_comparison)]
assert_eq!($transform(actual), $expected, "Wasm test failed")
}
}
};
($src:expr, $expected:expr, $ty:ty) => {
$crate::helpers::llvm::assert_wasm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::llvm::identity,
false
);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
$crate::helpers::llvm::assert_wasm_evals_to!($src, $expected, $ty, $transform, false);
};
}
#[allow(unused_macros)]
macro_rules! assert_llvm_evals_to {
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => {
use bumpalo::Bump;
use inkwell::context::Context;
use roc_gen_llvm::run_jit_function;
let arena = Bump::new();
let context = Context::create();
let is_gen_test = true;
let (main_fn_name, errors, lib) =
$crate::helpers::llvm::helper(&arena, $src, is_gen_test, $ignore_problems, &context);
let transform = |success| {
let expected = $expected;
#[allow(clippy::redundant_closure_call)]
let given = $transform(success);
assert_eq!(&given, &expected, "LLVM test failed");
};
run_jit_function!(lib, main_fn_name, $ty, transform, errors)
};
($src:expr, $expected:expr, $ty:ty) => {
$crate::helpers::llvm::assert_llvm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::llvm::identity,
false
);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
$crate::helpers::llvm::assert_llvm_evals_to!($src, $expected, $ty, $transform, false);
};
}
#[allow(unused_macros)]
macro_rules! assert_evals_to {
($src:expr, $expected:expr, $ty:ty) => {{
assert_evals_to!($src, $expected, $ty, $crate::helpers::llvm::identity, false);
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {{
// same as above, except with an additional transformation argument.
assert_evals_to!($src, $expected, $ty, $transform, false);
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{
// same as above, except with ignore_problems.
#[cfg(feature = "wasm-cli-run")]
$crate::helpers::llvm::assert_wasm_evals_to!(
$src,
$expected,
$ty,
$transform,
$ignore_problems
);
$crate::helpers::llvm::assert_llvm_evals_to!(
$src,
$expected,
$ty,
$transform,
$ignore_problems
);
}};
}
#[allow(unused_macros)]
macro_rules! assert_expect_failed {
($src:expr, $expected:expr, $ty:ty) => {
use bumpalo::Bump;
use inkwell::context::Context;
use roc_gen_llvm::run_jit_function;
let arena = Bump::new();
let context = Context::create();
let is_gen_test = true;
let (main_fn_name, errors, lib) =
$crate::helpers::llvm::helper(&arena, $src, is_gen_test, false, &context);
let transform = |success| {
let expected = $expected;
assert_eq!(&success, &expected, "LLVM test failed");
};
run_jit_function!(lib, main_fn_name, $ty, transform, errors)
};
($src:expr, $expected:expr, $ty:ty) => {
$crate::helpers::llvm::assert_llvm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::llvm::identity,
false
);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
$crate::helpers::llvm::assert_llvm_evals_to!($src, $expected, $ty, $transform, false);
};
}
macro_rules! expect_runtime_error_panic {
($src:expr) => {{
#[cfg(feature = "wasm-cli-run")]
$crate::helpers::llvm::assert_wasm_evals_to!(
$src,
false, // fake value/type for eval
bool,
$crate::helpers::llvm::identity,
true // ignore problems
);
$crate::helpers::llvm::assert_llvm_evals_to!(
$src,
false, // fake value/type for eval
bool,
$crate::helpers::llvm::identity,
true // ignore problems
);
}};
}
#[allow(dead_code)]
pub fn identity<T>(value: T) -> T {
value
}
#[allow(unused_macros)]
macro_rules! assert_non_opt_evals_to {
($src:expr, $expected:expr, $ty:ty) => {{
$crate::helpers::llvm::assert_llvm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::llvm::identity
);
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
// Same as above, except with an additional transformation argument.
{
$crate::helpers::llvm::assert_llvm_evals_to!($src, $expected, $ty, $transform);
}
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {{
$crate::helpers::llvm::assert_llvm_evals_to!($src, $expected, $ty, $transform);
}};
}
#[allow(unused_imports)]
pub(crate) use assert_evals_to;
#[allow(unused_imports)]
pub(crate) use assert_expect_failed;
#[allow(unused_imports)]
pub(crate) use assert_llvm_evals_to;
#[allow(unused_imports)]
pub(crate) use assert_non_opt_evals_to;
#[allow(unused_imports)]
pub(crate) use assert_wasm_evals_to;
#[allow(unused_imports)]
pub(crate) use expect_runtime_error_panic;

65
crates/compiler/test_gen/src/helpers/mod.rs Normal file
View file

@ -0,0 +1,65 @@
extern crate bumpalo;
#[cfg(feature = "gen-dev")]
pub mod dev;
pub mod from_wasmer_memory;
#[cfg(feature = "gen-llvm")]
pub mod llvm;
#[cfg(feature = "gen-wasm")]
pub mod wasm;
#[allow(dead_code)]
pub fn zig_executable() -> String {
match std::env::var("ROC_ZIG") {
Ok(path) => path,
Err(_) => "zig".into(),
}
}
/// Used in the with_larger_debug_stack() function, for tests that otherwise
/// run out of stack space in debug builds (but don't in --release builds)
#[allow(dead_code)]
const EXPANDED_STACK_SIZE: usize = 8 * 1024 * 1024;
/// Without this, some tests pass in `cargo test --release` but fail without
/// the --release flag because they run out of stack space. This increases
/// stack size for debug builds only, while leaving the stack space at the default
/// amount for release builds.
#[allow(dead_code)]
#[cfg(debug_assertions)]
pub fn with_larger_debug_stack<F>(run_test: F)
where
F: FnOnce(),
F: Send,
F: 'static,
{
std::thread::Builder::new()
.stack_size(EXPANDED_STACK_SIZE)
.spawn(run_test)
.expect("Error while spawning expanded dev stack size thread")
.join()
.expect("Error while joining expanded dev stack size thread")
}
/// In --release builds, don't increase the stack size. Run the test normally.
/// This way, we find out if any of our tests are blowing the stack even after
/// optimizations in release builds.
#[allow(dead_code)]
#[cfg(not(debug_assertions))]
#[inline(always)]
pub fn with_larger_debug_stack<F>(run_test: F)
where
F: FnOnce(),
F: Send,
F: 'static,
{
run_test()
}
#[allow(dead_code)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RefCount {
Live(u32),
Deallocated,
Constant,
}

426
crates/compiler/test_gen/src/helpers/wasm.rs Normal file
View file

@ -0,0 +1,426 @@
use super::RefCount;
use crate::helpers::from_wasmer_memory::FromWasmerMemory;
use roc_collections::all::MutSet;
use roc_gen_wasm::wasm32_result::Wasm32Result;
use roc_gen_wasm::wasm_module::{Export, ExportType};
use roc_gen_wasm::{DEBUG_SETTINGS, MEMORY_NAME};
use roc_load::Threading;
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use std::marker::PhantomData;
use std::path::{Path, PathBuf};
use wasmer::{Memory, WasmPtr};
// Should manually match build.rs
const PLATFORM_FILENAME: &str = "wasm_test_platform";
const OUT_DIR_VAR: &str = "TEST_GEN_OUT";
const TEST_WRAPPER_NAME: &str = "test_wrapper";
const INIT_REFCOUNT_NAME: &str = "init_refcount_test";
const PANIC_MSG_NAME: &str = "panic_msg";
fn promote_expr_to_module(src: &str) -> String {
let mut buffer = String::from("app \"test\" provides [main] to \"./platform\"\n\nmain =\n");
for line in src.lines() {
// indent the body!
buffer.push_str(" ");
buffer.push_str(line);
buffer.push('\n');
}
buffer
}
#[allow(dead_code)]
pub fn compile_and_load<'a, T: Wasm32Result>(
arena: &'a bumpalo::Bump,
src: &str,
test_wrapper_type_info: PhantomData<T>,
) -> wasmer::Instance {
let platform_path = get_preprocessed_host_path();
let platform_bytes = std::fs::read(&platform_path).unwrap();
println!("Loading test host {}", platform_path.display());
let compiled_bytes =
compile_roc_to_wasm_bytes(arena, &platform_bytes, src, test_wrapper_type_info);
if DEBUG_SETTINGS.keep_test_binary {
let build_dir_hash = src_hash(src);
save_wasm_file(&compiled_bytes, build_dir_hash)
};
load_bytes_into_runtime(&compiled_bytes)
}
fn get_preprocessed_host_path() -> PathBuf {
let out_dir = std::env::var(OUT_DIR_VAR).unwrap();
Path::new(&out_dir)
.join([PLATFORM_FILENAME, "o"].join("."))
.to_path_buf()
}
fn src_hash(src: &str) -> u64 {
let mut hash_state = DefaultHasher::new();
src.hash(&mut hash_state);
hash_state.finish()
}
fn compile_roc_to_wasm_bytes<'a, T: Wasm32Result>(
arena: &'a bumpalo::Bump,
host_bytes: &[u8],
src: &str,
_test_wrapper_type_info: PhantomData<T>,
) -> Vec<u8> {
let filename = PathBuf::from("Test.roc");
let src_dir = Path::new("fake/test/path");
let module_src;
let temp;
if src.starts_with("app") {
// this is already a module
module_src = src;
} else {
// this is an expression, promote it to a module
temp = promote_expr_to_module(src);
module_src = &temp;
}
let loaded = roc_load::load_and_monomorphize_from_str(
arena,
filename,
module_src,
src_dir,
Default::default(),
roc_target::TargetInfo::default_wasm32(),
roc_reporting::report::RenderTarget::ColorTerminal,
Threading::Single,
);
let loaded = loaded.expect("failed to load module");
use roc_load::MonomorphizedModule;
let MonomorphizedModule {
module_id,
procedures,
mut interns,
exposed_to_host,
..
} = loaded;
debug_assert_eq!(exposed_to_host.values.len(), 1);
let exposed_to_host = exposed_to_host
.values
.keys()
.copied()
.collect::<MutSet<_>>();
let env = roc_gen_wasm::Env {
arena,
module_id,
exposed_to_host,
};
let host_module = roc_gen_wasm::parse_host(env.arena, host_bytes).unwrap_or_else(|e| {
panic!(
"I ran into a problem with the host object file, {} at offset 0x{:x}:\n{}",
get_preprocessed_host_path().display(),
e.offset,
e.message
)
});
let (mut module, called_preload_fns, main_fn_index) =
roc_gen_wasm::build_app_module(&env, &mut interns, host_module, procedures);
T::insert_wrapper(arena, &mut module, TEST_WRAPPER_NAME, main_fn_index);
// Export the initialiser function for refcount tests
let init_refcount_idx = module
.names
.function_names
.iter()
.filter(|(_, name)| *name == INIT_REFCOUNT_NAME)
.map(|(i, _)| *i)
.next()
.unwrap();
module.export.append(Export {
name: INIT_REFCOUNT_NAME,
ty: ExportType::Func,
index: init_refcount_idx,
});
module.eliminate_dead_code(env.arena, called_preload_fns);
let mut app_module_bytes = std::vec::Vec::with_capacity(module.size());
module.serialize(&mut app_module_bytes);
app_module_bytes
}
fn save_wasm_file(app_module_bytes: &[u8], build_dir_hash: u64) {
let debug_dir_str = format!("/tmp/roc/gen_wasm/{:016x}", build_dir_hash);
let debug_dir_path = Path::new(&debug_dir_str);
let final_wasm_file = debug_dir_path.join("final.wasm");
std::fs::create_dir_all(debug_dir_path).unwrap();
std::fs::write(&final_wasm_file, app_module_bytes).unwrap();
println!(
"Debug command:\n\twasm-objdump -dx {}",
final_wasm_file.to_str().unwrap()
);
}
fn load_bytes_into_runtime(bytes: &[u8]) -> wasmer::Instance {
use wasmer::{Module, Store};
use wasmer_wasi::WasiState;
let store = Store::default();
let wasmer_module = Module::new(&store, bytes).unwrap();
// First, we create the `WasiEnv`
let mut wasi_env = WasiState::new("hello").finalize().unwrap();
// Then, we get the import object related to our WASI
// and attach it to the Wasm instance.
let import_object = wasi_env
.import_object(&wasmer_module)
.unwrap_or_else(|_| wasmer::imports!());
wasmer::Instance::new(&wasmer_module, &import_object).unwrap()
}
#[allow(dead_code)]
pub fn assert_evals_to_help<T>(src: &str, phantom: PhantomData<T>) -> Result<T, String>
where
T: FromWasmerMemory + Wasm32Result,
{
let arena = bumpalo::Bump::new();
let instance = crate::helpers::wasm::compile_and_load(&arena, src, phantom);
let memory = instance.exports.get_memory(MEMORY_NAME).unwrap();
let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap();
match test_wrapper.call(&[]) {
Err(e) => {
if let Some(msg) = get_roc_panic_msg(&instance, memory) {
Err(format!("Roc failed with message: \"{}\"", msg))
} else {
Err(e.to_string())
}
}
Ok(result) => {
let address = result[0].unwrap_i32();
if false {
println!("test_wrapper returned 0x{:x}", address);
println!("Stack:");
crate::helpers::wasm::debug_memory_hex(memory, address, std::mem::size_of::<T>());
}
if false {
println!("Heap:");
// Manually provide address and size based on printf in wasm_test_platform.c
crate::helpers::wasm::debug_memory_hex(memory, 0x11440, 24);
}
let output = <T as FromWasmerMemory>::decode(memory, address as u32);
Ok(output)
}
}
}
/// Our test roc_panic stores a pointer to its message in a global variable so we can find it.
fn get_roc_panic_msg(instance: &wasmer::Instance, memory: &Memory) -> Option<String> {
let memory_bytes = unsafe { memory.data_unchecked() };
// We need to dereference twice!
// The Wasm Global only points at the memory location of the C global value
let panic_msg_global = instance.exports.get_global(PANIC_MSG_NAME).unwrap();
let global_addr = panic_msg_global.get().unwrap_i32() as usize;
let global_ptr = memory_bytes[global_addr..].as_ptr() as *const u32;
// Dereference again to find the bytes of the message string
let msg_addr = unsafe { *global_ptr };
if msg_addr == 0 {
return None;
}
let msg_index = msg_addr as usize;
let msg_len = memory_bytes[msg_index..]
.iter()
.position(|c| *c == 0)
.unwrap();
let msg_bytes = memory_bytes[msg_index..][..msg_len].to_vec();
let msg = unsafe { String::from_utf8_unchecked(msg_bytes) };
Some(msg)
}
#[allow(dead_code)]
pub fn assert_wasm_refcounts_help<T>(
src: &str,
phantom: PhantomData<T>,
num_refcounts: usize,
) -> Result<Vec<RefCount>, String>
where
T: FromWasmerMemory + Wasm32Result,
{
let arena = bumpalo::Bump::new();
let instance = crate::helpers::wasm::compile_and_load(&arena, src, phantom);
let memory = instance.exports.get_memory(MEMORY_NAME).unwrap();
let expected_len = num_refcounts as i32;
let init_refcount_test = instance.exports.get_function(INIT_REFCOUNT_NAME).unwrap();
let init_result = init_refcount_test.call(&[wasmer::Value::I32(expected_len)]);
let refcount_vector_addr = match init_result {
Err(e) => return Err(format!("{:?}", e)),
Ok(result) => result[0].unwrap_i32(),
};
// Run the test
let test_wrapper = instance.exports.get_function(TEST_WRAPPER_NAME).unwrap();
match test_wrapper.call(&[]) {
Err(e) => return Err(format!("{:?}", e)),
Ok(_) => {}
}
// Check we got the right number of refcounts
let refcount_vector_len: WasmPtr<i32> = WasmPtr::new(refcount_vector_addr as u32);
let actual_len = refcount_vector_len.deref(memory).unwrap().get();
if actual_len != expected_len {
panic!("Expected {} refcounts but got {}", expected_len, actual_len);
}
// Read the actual refcount values
let refcount_ptr_array: WasmPtr<WasmPtr<i32>, wasmer::Array> =
WasmPtr::new(4 + refcount_vector_addr as u32);
let refcount_ptrs = refcount_ptr_array
.deref(memory, 0, num_refcounts as u32)
.unwrap();
let mut refcounts = Vec::with_capacity(num_refcounts);
for i in 0..num_refcounts {
let rc_ptr = refcount_ptrs[i].get();
let rc = if rc_ptr.offset() == 0 {
RefCount::Deallocated
} else {
let rc_encoded: i32 = rc_ptr.deref(memory).unwrap().get();
if rc_encoded == 0 {
RefCount::Constant
} else {
let rc = rc_encoded - i32::MIN + 1;
RefCount::Live(rc as u32)
}
};
refcounts.push(rc);
}
Ok(refcounts)
}
/// Print out hex bytes of the test result, and a few words on either side
/// Can be handy for debugging misalignment issues etc.
pub fn debug_memory_hex(memory: &Memory, address: i32, size: usize) {
let memory_words: &[u32] = unsafe {
let memory_bytes = memory.data_unchecked();
std::mem::transmute(memory_bytes)
};
let extra_words = 2;
let result_start = (address as usize) / 4;
let result_end = result_start + ((size + 3) / 4);
let start = result_start - extra_words;
let end = result_end + extra_words;
for index in start..end {
let result_marker = if index >= result_start && index < result_end {
"|"
} else {
" "
};
println!(
"{:x} {} {:08x}",
index * 4,
result_marker,
memory_words[index]
);
}
println!();
}
#[allow(unused_macros)]
macro_rules! assert_evals_to {
($src:expr, $expected:expr, $ty:ty) => {
$crate::helpers::wasm::assert_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::wasm::identity,
false
)
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
$crate::helpers::wasm::assert_evals_to!($src, $expected, $ty, $transform, false);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems: expr) => {{
let phantom = std::marker::PhantomData;
let _ = $ignore_problems; // Always ignore "problems"! One backend (LLVM) is enough to cover them.
match $crate::helpers::wasm::assert_evals_to_help::<$ty>($src, phantom) {
Err(msg) => panic!("{}", msg),
Ok(actual) => {
assert_eq!($transform(actual), $expected)
}
}
}};
}
#[allow(unused_macros)]
macro_rules! expect_runtime_error_panic {
($src:expr) => {{
$crate::helpers::wasm::assert_evals_to!(
$src,
false, // fake value/type for eval
bool,
$crate::helpers::wasm::identity,
true // ignore problems
);
}};
}
#[allow(dead_code)]
pub fn identity<T>(value: T) -> T {
value
}
#[allow(unused_macros)]
macro_rules! assert_refcounts {
// We need the result type to generate the test_wrapper, even though we ignore the value!
// We can't just call `main` with no args, because some tests return structs, via pointer arg!
// Also we need to know how much stack space to reserve for the struct.
($src: expr, $ty: ty, $expected_refcounts: expr) => {{
let phantom = std::marker::PhantomData;
let num_refcounts = $expected_refcounts.len();
let result =
$crate::helpers::wasm::assert_wasm_refcounts_help::<$ty>($src, phantom, num_refcounts);
match result {
Err(msg) => panic!("{:?}", msg),
Ok(actual_refcounts) => {
assert_eq!(&actual_refcounts, $expected_refcounts)
}
}
}};
}
#[allow(unused_imports)]
pub(crate) use assert_evals_to;
#[allow(unused_imports)]
pub(crate) use expect_runtime_error_panic;
#[allow(unused_imports)]
pub(crate) use assert_refcounts;

26
crates/compiler/test_gen/src/helpers/wasm_linking_host_imports.zig Normal file
View file

@ -0,0 +1,26 @@
// Definitions to allow us to run an all-Zig version of the linking test
// Allows us to calculate the expected answer!
extern fn host_called_directly_from_roc() i32;
export var host_result: i32 = 0;
export fn js_called_directly_from_roc() i32 {
return 0x01;
}
export fn js_called_indirectly_from_roc() i32 {
return 0x02;
}
export fn js_called_directly_from_main() i32 {
return 0x04;
}
export fn js_called_indirectly_from_main() i32 {
return 0x08;
}
export fn js_unused() i32 {
return 0x10;
}
export fn roc__app_proc_1_exposed() i32 {
return 0x20 | js_called_directly_from_roc() | host_called_directly_from_roc();
}

44
crates/compiler/test_gen/src/helpers/wasm_linking_test_host.zig Normal file
View file

@ -0,0 +1,44 @@
extern fn js_called_directly_from_roc() i32;
extern fn js_called_indirectly_from_roc() i32;
extern fn js_called_directly_from_main() i32;
extern fn js_called_indirectly_from_main() i32;
extern fn js_unused() i32;
extern fn roc__app_proc_1_exposed() i32;
export fn host_called_indirectly_from_roc() i32 {
return 0x40;
}
export fn host_called_directly_from_roc() i32 {
return 0x80 | host_called_indirectly_from_roc() | js_called_indirectly_from_roc();
}
export fn host_called_indirectly_from_main() i32 {
return 0x100;
}
export fn host_called_directly_from_main() i32 {
return 0x200 | host_called_indirectly_from_main() | js_called_indirectly_from_main();
}
export fn host_unused() i32 {
// Call some functions from here to get them included in the output file
return 0x400 | js_unused() | js_called_directly_from_roc();
}
// Result is an extern global so the test can read it from the Wasm module
extern var host_result: i32;
pub fn main() !void {
const host = host_called_directly_from_main();
const js = js_called_directly_from_main();
const app = roc__app_proc_1_exposed();
host_result = host | js | app;
if (@import("builtin").target.cpu.arch != .wasm32) {
const stdout = @import("std").io.getStdOut().writer();
try stdout.print("{}\n", .{host_result});
}
}

152
crates/compiler/test_gen/src/helpers/wasm_test_platform.c Normal file
View file

@ -0,0 +1,152 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
// Makes test runs take 50% longer, due to linking
#define ENABLE_PRINTF 0
typedef struct
{
size_t length;
size_t *elements[]; // flexible array member
} Vector;
// Globals for refcount testing
Vector *rc_pointers;
size_t rc_pointers_capacity;
// The rust test passes us the max number of allocations it expects to make,
// and we tell it where we're going to write the refcount pointers.
// It won't actually read that memory until later, when the test is done.
Vector *init_refcount_test(size_t capacity)
{
rc_pointers_capacity = capacity;
rc_pointers = malloc((1 + capacity) * sizeof(size_t *));
rc_pointers->length = 0;
for (size_t i = 0; i < capacity; ++i)
rc_pointers->elements[i] = NULL;
return rc_pointers;
}
#if ENABLE_PRINTF
#define ASSERT(condition, format, ...) \
if (!(condition)) \
{ \
printf("ASSERT FAILED: " #format "\n", __VA_ARGS__); \
abort(); \
}
#else
#define ASSERT(condition, format, ...) \
if (!(condition)) \
abort();
#endif
size_t *alloc_ptr_to_rc_ptr(void *ptr, unsigned int alignment)
{
size_t alloc_addr = (size_t)ptr;
size_t rc_addr = alloc_addr + alignment - sizeof(size_t);
return (size_t *)rc_addr;
}
//--------------------------
void *roc_alloc(size_t size, unsigned int alignment)
{
void *allocated = malloc(size);
if (rc_pointers)
{
ASSERT(alignment >= sizeof(size_t), "alignment %zd != %zd", alignment, sizeof(size_t));
size_t num_alloc = rc_pointers->length + 1;
ASSERT(num_alloc <= rc_pointers_capacity, "Too many allocations %zd > %zd", num_alloc, rc_pointers_capacity);
size_t *rc_ptr = alloc_ptr_to_rc_ptr(allocated, alignment);
rc_pointers->elements[rc_pointers->length] = rc_ptr;
rc_pointers->length++;
}
#if ENABLE_PRINTF
if (!allocated)
{
fprintf(stderr, "roc_alloc failed\n");
exit(1);
}
else
{
printf("roc_alloc allocated %d bytes with alignment %d at %p\n", size, alignment, allocated);
}
#endif
return allocated;
}
//--------------------------
void *roc_realloc(void *ptr, size_t new_size, size_t old_size,
unsigned int alignment)
{
#if ENABLE_PRINTF
printf("roc_realloc reallocated %p from %d to %d with alignment %zd\n",
ptr, old_size, new_size, alignment);
#endif
return realloc(ptr, new_size);
}
//--------------------------
void roc_dealloc(void *ptr, unsigned int alignment)
{
if (rc_pointers)
{
// Null out the entry in the test array to indicate that it was freed
// Then even if malloc reuses the space, everything still works
size_t *rc_ptr = alloc_ptr_to_rc_ptr(ptr, alignment);
int i = 0;
for (; i < rc_pointers->length; ++i)
{
if (rc_pointers->elements[i] == rc_ptr)
{
rc_pointers->elements[i] = NULL;
break;
}
}
int was_found = i < rc_pointers->length;
ASSERT(was_found, "RC pointer not found %p", rc_ptr);
}
#if ENABLE_PRINTF
printf("roc_dealloc deallocated %p with alignment %zd\n", ptr, alignment);
#endif
free(ptr);
}
//--------------------------
// Allow the test to probe the panic message
extern char* panic_msg;
void roc_panic(char *msg, unsigned int tag_id)
{
panic_msg = msg;
// Note: no dynamic string formatting
fputs("Application crashed with message\n\n ", stderr);
fputs(msg, stderr);
fputs("\n\nShutting down\n", stderr);
exit(101);
}
//--------------------------
void roc_memcpy(void *dest, const void *src, size_t n)
{
memcpy(dest, src, n);
}
//--------------------------
void *roc_memset(void *str, int c, size_t n)
{
return memset(str, c, n);
}