Merge pull request #2983 from rtfeldman/wasi-libc-sys

Build WASI libc using Zig, instead of using a checked-in binary

Cargo.lock

@@ -3462,6 +3462,7 @@ dependencies = [
  "serde_json",
  "target-lexicon",
  "tempfile",
+ "wasi_libc_sys",
 ]
 
 [[package]]
@@ -5028,6 +5029,10 @@ version = "0.10.2+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
 
+[[package]]
+name = "wasi_libc_sys"
+version = "0.1.0"
+
 [[package]]
 name = "wasm-bindgen"
 version = "0.2.79"

Cargo.toml

@@ -46,6 +46,7 @@ members = [
     "utils",
     "docs",
     "linker",
+    "wasi-libc-sys",
 ]
 exclude = [
     "ci/bench-runner",

Earthfile

@@ -54,21 +54,26 @@ install-zig-llvm-valgrind-clippy-rustfmt:
 
 copy-dirs:
     FROM +install-zig-llvm-valgrind-clippy-rustfmt
-    COPY --dir cli cli_utils compiler docs editor ast code_markup error_macros highlight utils test_utils reporting repl_cli repl_eval repl_test repl_wasm repl_www roc_std vendor examples linker Cargo.toml Cargo.lock version.txt www ./
+    COPY --dir cli cli_utils compiler docs editor ast code_markup error_macros highlight utils test_utils reporting repl_cli repl_eval repl_test repl_wasm repl_www roc_std vendor examples linker Cargo.toml Cargo.lock version.txt www wasi-libc-sys ./
 
 test-zig:
     FROM +install-zig-llvm-valgrind-clippy-rustfmt
     COPY --dir compiler/builtins/bitcode ./
     RUN cd bitcode && ./run-tests.sh && ./run-wasm-tests.sh
 
-check-clippy:
+build-rust-test:
     FROM +copy-dirs
+    RUN --mount=type=cache,target=$SCCACHE_DIR \
+        cargo test --locked --release --features with_sound --workspace --no-run && sccache --show-stats
+
+check-clippy:
+    FROM +build-rust-test
     RUN cargo clippy -V
     RUN --mount=type=cache,target=$SCCACHE_DIR \
         cargo clippy -- -D warnings
 
 check-rustfmt:
-    FROM +copy-dirs
+    FROM +build-rust-test
     RUN cargo fmt --version
     RUN cargo fmt --all -- --check
 
@@ -78,7 +83,7 @@ check-typos:
     RUN typos
 
 test-rust:
-    FROM +copy-dirs
+    FROM +build-rust-test
     ENV RUST_BACKTRACE=1
     # for race condition problem with cli test
     ENV ROC_NUM_WORKERS=1

compiler/build/Cargo.toml

@@ -31,6 +31,7 @@ libloading = "0.7.1"
 tempfile = "3.2.0"
 inkwell = { path = "../../vendor/inkwell", optional = true }
 target-lexicon = "0.12.3"
+wasi_libc_sys = { path = "../../wasi-libc-sys" }
 
 [target.'cfg(target_os = "macos")'.dependencies]
 serde_json = "1.0.69"

compiler/build/src/link.rs

@@ -72,16 +72,12 @@ fn find_zig_str_path() -> PathBuf {
 }
 
 fn find_wasi_libc_path() -> PathBuf {
-    let wasi_libc_path = PathBuf::from("compiler/builtins/bitcode/wasi-libc.a");
+    use wasi_libc_sys::WASI_LIBC_PATH;
 
-    if std::path::Path::exists(&wasi_libc_path) {
-        return wasi_libc_path;
-    }
-
-    // when running the tests, we start in the /cli directory
-    let wasi_libc_path = PathBuf::from("../compiler/builtins/bitcode/wasi-libc.a");
-    if std::path::Path::exists(&wasi_libc_path) {
-        return wasi_libc_path;
+    // Environment variable defined in wasi-libc-sys/build.rs
+    let wasi_libc_pathbuf = PathBuf::from(WASI_LIBC_PATH);
+    if std::path::Path::exists(&wasi_libc_pathbuf) {
+        return wasi_libc_pathbuf;
     }
 
     panic!("cannot find `wasi-libc.a`")

compiler/test_gen/src/helpers/wasm.rs

@@ -36,12 +36,12 @@ fn promote_expr_to_module(src: &str) -> String {
 pub fn compile_and_load<'a, T: Wasm32Result>(
     arena: &'a bumpalo::Bump,
     src: &str,
-    _test_wrapper_type_info: PhantomData<T>,
+    test_wrapper_type_info: PhantomData<T>,
 ) -> wasmer::Instance {
     let platform_bytes = load_platform_and_builtins();
 
     let compiled_bytes =
-        compile_roc_to_wasm_bytes(arena, &platform_bytes, src, _test_wrapper_type_info);
+        compile_roc_to_wasm_bytes(arena, &platform_bytes, src, test_wrapper_type_info);
 
     if DEBUG_LOG_SETTINGS.keep_test_binary {
         let build_dir_hash = src_hash(src);

wasi-libc-sys/.gitignore

@@ -0,0 +1 @@
+src/generated.rs

wasi-libc-sys/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+authors = ["The Roc Contributors"]
+description = "Rust wrapper for a WebAssembly test platform built on libc"
+edition = "2018"
+license = "UPL-1.0"
+name = "wasi_libc_sys"
+repository = "https://github.com/rtfeldman/roc"
+version = "0.1.0"

wasi-libc-sys/build.rs

@@ -0,0 +1,75 @@
+use std::env;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-changed=src/dummy.c");
+
+    let out_dir = env::var("OUT_DIR").unwrap();
+    let zig_cache_dir = format!("{}/zig-cache", out_dir);
+    let out_file = format!("{}/wasi-libc.a", out_dir);
+
+    // Compile a dummy C program with Zig, with our own private cache directory
+    let zig = zig_executable();
+    run_command(
+        Path::new("."),
+        &zig,
+        [
+            "build-exe",
+            "-target",
+            "wasm32-wasi",
+            "-lc",
+            "-O",
+            "ReleaseSmall",
+            "--global-cache-dir",
+            &zig_cache_dir,
+            "src/dummy.c",
+            &format!("-femit-bin={}/dummy.wasm", out_dir),
+        ],
+    );
+
+    // Find the libc.a file that Zig wrote (as a side-effect of compiling the dummy program)
+    let find_cmd_output = run_command(Path::new("."), "find", [&zig_cache_dir, "-name", "libc.a"]);
+    let zig_libc_path = find_cmd_output.trim(); // get rid of a newline
+
+    // Copy libc to where Cargo expects it
+    fs::copy(&zig_libc_path, &out_file).unwrap();
+
+    // Generate some Rust code to indicate where the file is
+    let generated_rust = format!("pub const WASI_LIBC_PATH: &str =\n    \"{}\";\n", out_file);
+    fs::write("src/generated.rs", generated_rust).unwrap();
+}
+
+fn zig_executable() -> String {
+    match std::env::var("ROC_ZIG") {
+        Ok(path) => path,
+        Err(_) => "zig".into(),
+    }
+}
+
+fn run_command<S, I, P: AsRef<Path>>(path: P, command_str: &str, args: I) -> String
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<OsStr>,
+{
+    let output_result = Command::new(OsStr::new(&command_str))
+        .current_dir(path)
+        .args(args)
+        .output();
+    match output_result {
+        Ok(output) => match output.status.success() {
+            true => std::str::from_utf8(&output.stdout).unwrap().to_string(),
+            false => {
+                let error_str = match std::str::from_utf8(&output.stderr) {
+                    Ok(stderr) => stderr.to_string(),
+                    Err(_) => format!("Failed to run \"{}\"", command_str),
+                };
+                panic!("{} failed: {}", command_str, error_str);
+            }
+        },
+        Err(reason) => panic!("{} failed: {}", command_str, reason),
+    }
+}

wasi-libc-sys/src/dummy.c

@@ -0,0 +1,6 @@
+#include <stdio.h>
+
+int main(int argc, char **argv)
+{
+    printf("I am a dummy C program. I only exist to help the build script to find libc.a and copy it");
+}

wasi-libc-sys/src/lib.rs

@@ -0,0 +1,18 @@
+// Rust's libc crate doesn't support Wasm, so we provide an implementation from Zig
+// We define Rust signatures here as we need them, rather than trying to cover all of libc
+#[cfg(target_family = "wasm")]
+use core::ffi::c_void;
+#[cfg(target_family = "wasm")]
+extern "C" {
+    pub fn malloc(size: usize) -> *mut c_void;
+    pub fn free(p: *mut c_void);
+    pub fn realloc(p: *mut c_void, size: usize) -> *mut c_void;
+    pub fn memcpy(dst: *mut c_void, src: *mut c_void, n: usize) -> *mut c_void;
+    pub fn memset(dst: *mut c_void, ch: i32, n: usize) -> *mut c_void;
+}
+
+// Tell users of this crate where to find the Wasm .a file
+// If a non-Wasm target is using this crate, we assume it is a build script that wants to emit Wasm
+// For Wasm target, it won't ever be used, but we expose it just to keep things simple
+mod generated;
+pub use generated::WASI_LIBC_PATH;