refactoring progress, use valgrind on exe

crates/cli_test_utils/src/command.rs

@@ -0,0 +1,166 @@
+
+use std::{ffi::OsString, process::{Command, ExitStatus, Stdio}};
+use std::io::Write;
+
+use regex::Regex;
+use roc_command_utils::pretty_command_string;
+use roc_reporting::report::ANSI_STYLE_CODES;
+
+pub fn run_command(mut cmd: Command, stdin_vals:&[&str]) -> CmdOut {
+    let cmd_str = pretty_command_string(&cmd);
+
+    let command = cmd
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped());
+
+    let mut roc_cmd_child = command.spawn().unwrap_or_else(|err| {
+        panic!("Failed to execute command:\n\n  {cmd_str:?}\n\nwith error:\n\n  {err}",)
+    });
+
+    let stdin = roc_cmd_child.stdin.as_mut().expect("Failed to open stdin");
+
+    for stdin_str in stdin_vals.iter() {
+        stdin
+            .write_all(stdin_str.as_bytes())
+            .unwrap_or_else(|err| {
+                panic!("Failed to write to stdin for command\n\n  {cmd_str:?}\n\nwith error:\n\n  {err}")
+            });
+    }
+    let roc_cmd_output = roc_cmd_child.wait_with_output().unwrap_or_else(|err| {
+        panic!("Failed to get output for command\n\n  {cmd_str:?}\n\nwith error:\n\n  {err}")
+    });
+
+    CmdOut {
+        stdout: String::from_utf8(roc_cmd_output.stdout).unwrap(),
+        stderr: String::from_utf8(roc_cmd_output.stderr).unwrap(),
+        status: roc_cmd_output.status,
+        cmd_str,
+    }
+}
+
+#[derive(Debug)]
+pub struct CmdOut {
+    pub stdout: String,
+    pub stderr: String,
+    pub status: ExitStatus,
+    pub cmd_str: OsString, // command with all its arguments, for easy debugging
+}
+
+impl std::fmt::Display for CmdOut {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "Command: {}\n\nExit Code: {}\n\nStdout:\n{}\n\nStderr:\n{}",
+            self.cmd_str.to_str().unwrap(),
+            self.status,
+            self.stdout,
+            self.stderr
+        )
+    }
+}
+
+impl CmdOut {
+    /// Assert that the command succeeded and that there are no unexpected errors in the stderr.
+    pub fn assert_clean_success(&self) {
+        self.assert_success_with_no_unexpected_errors();
+    }
+
+    pub fn assert_nonzero_exit(&self) {
+        assert!(!self.status.success());
+    }
+
+    /// Assert that the command succeeded and that there are no unexpected errors in the stderr.
+    /// This DOES NOT normalise the output, use assert_stdout_ends_with for that.
+    pub fn assert_success_with_no_unexpected_errors(&self) {
+        assert!(self.status.success(), "Command failed\n\n{self}");
+        assert_no_unexpected_error(&self.stderr);
+    }
+    
+    pub fn assert_clean_stdout(&self, expected_out: &str) {
+        self.assert_clean_success();
+        
+        let normalised_output = normalize_for_tests(&self.stdout);
+        
+        assert_eq!(normalised_output, expected_out);
+    }
+
+    /// Assert that the stdout ends with the expected string
+    /// This normalises the output for comparison in tests such as replacing timings
+    /// with a placeholder, or stripping ANSI colors
+    pub fn assert_stdout_and_stderr_ends_with(&self, expected: &str) {
+        let normalised_output = format!(
+            "{}{}",
+            normalize_for_tests(&self.stdout),
+            normalize_for_tests(&self.stderr)
+        );
+
+        assert!(
+            normalised_output.ends_with(expected),
+            "\n{}EXPECTED stdout and stderr after normalizing:\n----------------\n{}{}\n{}ACTUAL stdout and stderr after normalizing:\n----------------\n{}{}{}\n----------------\n{}",
+            ANSI_STYLE_CODES.cyan,
+            ANSI_STYLE_CODES.reset,
+            expected,
+            ANSI_STYLE_CODES.cyan,
+            ANSI_STYLE_CODES.reset,
+            normalised_output,
+            ANSI_STYLE_CODES.cyan,
+            ANSI_STYLE_CODES.reset,
+        );
+    }
+}
+
+pub fn assert_no_unexpected_error(stderr: &str) {
+    
+    let mut stderr_part_clean = String::from(stderr);
+    
+    let expected_errors = [
+        "🔨 Building host ...\n",
+        "ld: warning: -undefined dynamic_lookup may not work with chained fixups",
+        "warning: ignoring debug info with an invalid version (0) in app\r\n",
+    ];
+    for expected_error in expected_errors {
+        stderr_part_clean = stderr_part_clean.replacen(&expected_error, "", 1);
+    }
+    
+    let clean_stderr = stderr_part_clean.trim();
+    
+    assert!(
+        clean_stderr.is_empty(),
+        "Unexpected error:\n{}",
+        clean_stderr
+    );
+}
+
+use lazy_static::lazy_static;
+
+/// Replaces things that are annoying to compare in tests with placeholders.
+fn normalize_for_tests(original_output: &str) -> String {
+    // normalise from windows line endings to unix line endings
+    let mut part_normalized = original_output.replace("\r\n", "\n");
+
+    // remove ANSI color codes
+    part_normalized = roc_reporting::report::strip_colors(&part_normalized);
+
+    // replace timings with a placeholder
+    let replacement = " in <ignored for test> ms.";
+    part_normalized = TIMING_REGEX.replace_all(&part_normalized, replacement).into_owned();
+
+    // replace file paths with a placeholder
+    let filepath_replacement = "[<ignored for tests>:$2]";
+    part_normalized = FILEPATH_REGEX.replace_all(&part_normalized, filepath_replacement).into_owned();
+
+    // replace error summary timings
+    let error_summary_replacement = "$1 error and $2 warning found in <ignored for test> ms";
+    ERROR_SUMMARY_REGEX
+        .replace_all(&part_normalized, error_summary_replacement)
+        .into_owned()
+}
+
+// Used by ^normalize_for_tests^
+lazy_static! {
+    static ref TIMING_REGEX: Regex = Regex::new(r" in (\d+) ms\.").expect("Invalid regex pattern");
+    static ref FILEPATH_REGEX: Regex = Regex::new(r"\[([^:]+):(\d+)\]").expect("Invalid filepath regex pattern");
+    static ref ERROR_SUMMARY_REGEX: Regex = Regex::new(r"(\d+) error(?:s)? and (\d+) warning(?:s)? found in \d+ ms")
+        .expect("Invalid error summary regex pattern");
+}

crates/cli_test_utils/src/exec_cli.rs

@@ -0,0 +1,114 @@
+use std::ffi::{OsStr, OsString};
+use std::path::PathBuf;
+use std::process::{Command};
+
+use crate::command::{run_command, CmdOut};
+use crate::helpers::path_to_roc_binary;
+use const_format::concatcp;
+
+const LINKER_FLAG: &str = concatcp!("--", roc_cli::FLAG_LINKER, "=", "legacy");
+
+
+/// A builder for running the Roc CLI.
+///
+/// Unlike `std::process::Command`, this builder is clonable. This is necessary to be able to test a command with both linkers.
+#[derive(Debug, Clone)]
+pub struct ExecCli {
+    sub_command: &'static str, // build, dev, test...
+    roc_file_path: PathBuf,
+    args: Vec<OsString>,
+}
+
+impl ExecCli {
+    pub fn new(sub_command: &'static str, roc_file_path: PathBuf) -> Self {
+        Self {
+            sub_command,
+            roc_file_path,
+            args: vec![],
+        }
+    }
+
+    pub fn arg<S>(mut self, arg: S) -> Self
+    where
+        S: Into<OsString>,
+    {
+        self.args.push(arg.into());
+        self
+    }
+
+    pub fn add_args<I, S>(mut self, args: I) -> Self
+    where
+        I: IntoIterator<Item = S>,
+        S: AsRef<OsStr>,
+    {
+        for arg in args {
+            self = self.arg(&arg);
+        }
+        self
+    }
+
+    pub fn run(&self) -> CmdOut {
+        let mut roc_cli_command = Command::new(path_to_roc_binary());
+        
+        roc_cli_command.arg(self.sub_command);
+        roc_cli_command.arg(self.roc_file_path.clone());
+        roc_cli_command.args(&self.args);
+        
+        run_command(roc_cli_command, &[])
+    }
+    
+    pub fn full_check(mut self, expected_output: &'static str, both_linkers: bool, with_valgrind: bool) {
+        self.check_build_and_run(expected_output, with_valgrind);
+
+        if both_linkers {
+            self = self.arg(LINKER_FLAG);
+            self.check_build_and_run(expected_output, with_valgrind);
+        }
+    }
+    
+    fn check_build_and_run(&self, expected_output: &'static str, with_valgrind: bool) {
+        let build_cmd_out = self.run();
+        build_cmd_out.assert_clean_success();
+
+        let executable_output = self.run_executable(with_valgrind);
+        executable_output.assert_clean_success();
+        assert_eq!(executable_output.stdout, expected_output);
+    }
+    
+    // run executable produced by e.g. `roc build`
+    fn run_executable(&self, with_valgrind: bool) -> CmdOut {
+        let executable = self.get_executable();
+        
+        if with_valgrind {
+            let mut command = Command::new("valgrind");
+            
+            command.args(&[
+                "--leak-check=full",
+                "--error-exitcode=1",
+                "--errors-for-leak-kinds=all",
+                executable.to_str().unwrap(),
+            ]);
+            
+            run_command(command, &[])
+        } else {
+            let command = Command::new(executable);
+            run_command(command, &[])
+        }
+    }
+    
+    fn get_executable(&self) -> PathBuf {
+        let mut executable_path = self.roc_file_path.with_extension("");
+
+        if cfg!(windows) {
+            executable_path.set_extension("exe");
+        }
+
+        if executable_path.exists() {
+            executable_path
+        } else {
+            panic!("Executable {:?} does not exist.", executable_path)
+        }
+    }
+    
+    
+}

crates/cli_test_utils/src/helpers.rs

@@ -0,0 +1,107 @@
+extern crate bumpalo;
+extern crate roc_collections;
+extern crate roc_load;
+extern crate roc_module;
+extern crate tempfile;
+
+use roc_command_utils::{cargo, root_dir};
+use std::env;
+use std::path::PathBuf;
+
+pub fn path_to_roc_binary() -> PathBuf {
+    path_to_binary(if cfg!(windows) { "roc.exe" } else { "roc" })
+}
+
+pub fn path_to_binary(binary_name: &str) -> PathBuf {
+    // Adapted from https://github.com/volta-cli/volta/blob/cefdf7436a15af3ce3a38b8fe53bb0cfdb37d3dd/tests/acceptance/support/sandbox.rs#L680
+    // by the Volta Contributors - license information can be found in
+    // the LEGAL_DETAILS file in the root directory of this distribution.
+    //
+    // Thank you, Volta contributors!
+    let mut path = env::var_os("CARGO_BIN_PATH")
+            .map(PathBuf::from)
+            .or_else(|| {
+                env::current_exe().ok().map(|mut path| {
+                    path.pop();
+                    if path.ends_with("deps") {
+                        path.pop();
+                    }
+                    path
+                })
+            })
+            .unwrap_or_else(|| panic!("CARGO_BIN_PATH wasn't set, and couldn't be inferred from context. Can't run CLI tests."));
+
+    path.push(binary_name);
+
+    path
+}
+
+// If we don't already have a /target/release/roc, build it!
+pub fn build_roc_bin_cached() -> PathBuf {
+    let roc_binary_path = path_to_roc_binary();
+
+    if !roc_binary_path.exists() {
+        build_roc_bin(&[]);
+    }
+
+    roc_binary_path
+}
+
+pub fn build_roc_bin(extra_args: &[&str]) -> PathBuf {
+    let roc_binary_path = path_to_roc_binary();
+
+    // Remove the /target/release/roc part
+    let root_project_dir = roc_binary_path
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap()
+        .parent()
+        .unwrap();
+
+    // cargo build --bin roc
+    // (with --release iff the test is being built with --release)
+    let mut args = if cfg!(debug_assertions) {
+        vec!["build", "--bin", "roc"]
+    } else {
+        vec!["build", "--release", "--bin", "roc"]
+    };
+
+    args.extend(extra_args);
+
+    let mut cargo_cmd = cargo();
+
+    cargo_cmd.current_dir(root_project_dir).args(&args);
+
+    let cargo_cmd_str = format!("{cargo_cmd:?}");
+
+    let cargo_output = cargo_cmd.output().unwrap();
+
+    if !cargo_output.status.success() {
+        panic!(
+            "The following cargo command failed:\n\n  {}\n\n  stdout was:\n\n    {}\n\n  stderr was:\n\n    {}\n",
+            cargo_cmd_str,
+            String::from_utf8(cargo_output.stdout).unwrap(),
+            String::from_utf8(cargo_output.stderr).unwrap()
+        );
+    }
+
+    roc_binary_path
+}
+
+#[allow(dead_code)]
+pub fn dir_from_root(dir_name: &str) -> PathBuf {
+    let mut path = root_dir();
+
+    path.extend(dir_name.split('/')); // Make slashes cross-target
+
+    path
+}
+
+pub fn file_from_root(dir_name: &str, file_name: &str) -> PathBuf {
+    let mut path = dir_from_root(dir_name);
+
+    path.push(file_name);
+
+    path
+}

crates/cli_test_utils/src/lib.rs

@@ -0,0 +1,5 @@
+//! Provides shared code for cli tests, cli benchmarks, glue tests, valgrind crate.
+pub mod bench_utils;
+pub mod helpers;
+pub mod exec_cli;
+pub mod command;