#[macro_use] extern crate pretty_assertions; extern crate bumpalo; extern crate roc_collections; extern crate roc_load; extern crate roc_module; #[cfg(test)] mod cli_run { use cli_utils::helpers::{ example_file, examples_dir, extract_valgrind_errors, fixture_file, run_cmd, run_roc, run_with_valgrind, ValgrindError, ValgrindErrorXWhat, }; use serial_test::serial; use std::path::Path; #[cfg(not(debug_assertions))] use roc_collections::all::MutMap; #[cfg(not(target_os = "macos"))] const ALLOW_VALGRIND: bool = true; // Disallow valgrind on macOS by default, because it reports a ton // of false positives. For local development on macOS, feel free to // change this to true! #[cfg(target_os = "macos")] const ALLOW_VALGRIND: bool = false; #[derive(Debug, PartialEq, Eq)] struct Example<'a> { filename: &'a str, executable_filename: &'a str, stdin: &'a [&'a str], expected_ending: &'a str, use_valgrind: bool, } fn check_output_with_stdin( file: &Path, stdin: &[&str], executable_filename: &str, flags: &[&str], expected_ending: &str, use_valgrind: bool, ) { let compile_out = run_roc(&[&["build", file.to_str().unwrap()], flags].concat()); if !compile_out.stderr.is_empty() { panic!("{}", compile_out.stderr); } assert!(compile_out.status.success(), "bad status {:?}", compile_out); let out = if use_valgrind && ALLOW_VALGRIND { let (valgrind_out, raw_xml) = run_with_valgrind( stdin, &[file.with_file_name(executable_filename).to_str().unwrap()], ); if valgrind_out.status.success() { let memory_errors = extract_valgrind_errors(&raw_xml).unwrap_or_else(|err| { panic!("failed to parse the `valgrind` xml output. Error was:\n\n{:?}\n\nvalgrind xml was: \"{}\"\n\nvalgrind stdout was: \"{}\"\n\nvalgrind stderr was: \"{}\"", err, raw_xml, valgrind_out.stdout, valgrind_out.stderr); }); if !memory_errors.is_empty() { for error in memory_errors { let ValgrindError { kind, what: _, xwhat, } = error; println!("Valgrind Error: {}\n", kind); if let Some(ValgrindErrorXWhat { text, leakedbytes: _, leakedblocks: _, }) = xwhat { println!(" {}", text); } } panic!("Valgrind reported memory errors"); } } else { let exit_code = match valgrind_out.status.code() { Some(code) => format!("exit code {}", code), None => "no exit code".to_string(), }; panic!("`valgrind` exited with {}. valgrind stdout was: \"{}\"\n\nvalgrind stderr was: \"{}\"", exit_code, valgrind_out.stdout, valgrind_out.stderr); } valgrind_out } else { run_cmd( file.with_file_name(executable_filename).to_str().unwrap(), stdin, &[], ) }; if !&out.stdout.ends_with(expected_ending) { panic!( "expected output to end with {:?} but instead got {:#?}", expected_ending, out ); } assert!(out.status.success()); } /// This macro does two things. /// /// First, it generates and runs a separate test for each of the given /// Example expressions. Each of these should test a particular .roc file /// in the examples/ directory. /// /// Second, it generates an extra test which (non-recursively) traverses the /// examples/ directory and verifies that each of the .roc files in there /// has had a corresponding test generated in the previous step. This test /// will fail if we ever add a new .roc file to examples/ and forget to /// add a test for it here! macro_rules! examples { ($($test_name:ident:$name:expr => $example:expr,)+) => { $( #[test] fn $test_name() { let dir_name = $name; let example = $example; let file_name = example_file(dir_name, example.filename); // Check with and without optimizations check_output_with_stdin( &file_name, example.stdin, example.executable_filename, &[], example.expected_ending, example.use_valgrind, ); check_output_with_stdin( &file_name, example.stdin, example.executable_filename, &["--optimize"], example.expected_ending, example.use_valgrind, ); } )* #[test] #[cfg(not(debug_assertions))] fn all_examples_have_tests() { let mut all_examples: MutMap<&str, Example<'_>> = MutMap::default(); $( all_examples.insert($name, $example); )* check_for_tests("../examples", &mut all_examples); } } } // examples! macro format: // // "name-of-subdirectory-inside-examples-dir" => [ // test_name_1: Example { // ... // }, // test_name_2: Example { // ... // }, // ] examples! { hello_world:"hello-world" => Example { filename: "Hello.roc", executable_filename: "hello-world", stdin: &[], expected_ending:"Hello, World!\n", use_valgrind: true, }, hello_zig:"hello-zig" => Example { filename: "Hello.roc", executable_filename: "hello-world", stdin: &[], expected_ending:"Hello, World!\n", use_valgrind: true, }, hello_rust:"hello-rust" => Example { filename: "Hello.roc", executable_filename: "hello-world", stdin: &[], expected_ending:"Hello, World!\n", use_valgrind: true, }, quicksort:"quicksort" => Example { filename: "Quicksort.roc", executable_filename: "quicksort", stdin: &[], expected_ending: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2]\n", use_valgrind: true, }, // shared_quicksort:"shared-quicksort" => Example { // filename: "Quicksort.roc", // executable_filename: "quicksort", // stdin: &[], // expected_ending: "[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2]\n", // use_valgrind: true, // }, effect:"effect" => Example { filename: "Main.roc", executable_filename: "effect-example", stdin: &["hi there!"], expected_ending: "hi there!\n", use_valgrind: true, }, // tea:"tea" => Example { // filename: "Main.roc", // executable_filename: "tea-example", // stdin: &[], // expected_ending: "", // use_valgrind: true, // }, cli:"cli" => Example { filename: "Echo.roc", executable_filename: "echo", stdin: &["Giovanni\n", "Giorgio\n"], expected_ending: "Hi, Giovanni Giorgio!\n", use_valgrind: true, }, // custom_malloc:"custom-malloc" => Example { // filename: "Main.roc", // executable_filename: "custom-malloc-example", // stdin: &[], // expected_ending: "ms!\nThe list was small!\n", // use_valgrind: true, // }, // task:"task" => Example { // filename: "Main.roc", // executable_filename: "task-example", // stdin: &[], // expected_ending: "successfully wrote to file\n", // use_valgrind: true, // }, } macro_rules! benchmarks { ($($test_name:ident => $benchmark:expr,)+) => { $( #[test] #[cfg_attr(not(debug_assertions), serial(benchmark))] fn $test_name() { let benchmark = $benchmark; let file_name = examples_dir("benchmarks").join(benchmark.filename); // TODO fix QuicksortApp and RBTreeCk and then remove this! match benchmark.filename { "QuicksortApp.roc" | "RBTreeCk.roc" => { eprintln!("WARNING: skipping testing benchmark {} because the test is broken right now!", benchmark.filename); return; } _ => {} } // Check with and without optimizations check_output_with_stdin( &file_name, benchmark.stdin, benchmark.executable_filename, &[], benchmark.expected_ending, benchmark.use_valgrind, ); check_output_with_stdin( &file_name, benchmark.stdin, benchmark.executable_filename, &["--optimize"], benchmark.expected_ending, benchmark.use_valgrind, ); } )* #[test] #[cfg(not(debug_assertions))] fn all_benchmarks_have_tests() { let mut all_benchmarks: MutMap<&str, Example<'_>> = MutMap::default(); $( let benchmark = $benchmark; all_benchmarks.insert(benchmark.filename, benchmark); )* check_for_benchmarks("../examples/benchmarks", &mut all_benchmarks); } } } benchmarks! { nqueens => Example { filename: "NQueens.roc", executable_filename: "nqueens", stdin: &["6"], expected_ending: "4\n", use_valgrind: true, }, cfold => Example { filename: "CFold.roc", executable_filename: "cfold", stdin: &["3"], expected_ending: "11 & 11\n", use_valgrind: true, }, deriv => Example { filename: "Deriv.roc", executable_filename: "deriv", stdin: &["2"], expected_ending: "1 count: 6\n2 count: 22\n", use_valgrind: true, }, rbtree_ck => Example { filename: "RBTreeCk.roc", executable_filename: "rbtree-ck", stdin: &[], expected_ending: "Node Black 0 {} Empty Empty\n", use_valgrind: true, }, rbtree_insert => Example { filename: "RBTreeInsert.roc", executable_filename: "rbtree-insert", stdin: &[], expected_ending: "Node Black 0 {} Empty Empty\n", use_valgrind: true, }, rbtree_del => Example { filename: "RBTreeDel.roc", executable_filename: "rbtree-del", stdin: &["420"], expected_ending: "30\n", use_valgrind: true, }, astar => Example { filename: "TestAStar.roc", executable_filename: "test-astar", stdin: &[], expected_ending: "True\n", use_valgrind: false, }, base64 => Example { filename: "TestBase64.roc", executable_filename: "test-base64", stdin: &[], expected_ending: "encoded: SGVsbG8gV29ybGQ=\ndecoded: Hello World\n", use_valgrind: true, }, closure => Example { filename: "Closure.roc", executable_filename: "closure", stdin: &[], expected_ending: "", use_valgrind: true, }, quicksort_app => Example { filename: "QuicksortApp.roc", executable_filename: "quicksortapp", stdin: &[], expected_ending: "todo put the correct quicksort answer here", use_valgrind: true, }, } #[cfg(not(debug_assertions))] fn check_for_tests(examples_dir: &str, all_examples: &mut MutMap<&str, Example<'_>>) { let entries = std::fs::read_dir(examples_dir).unwrap_or_else(|err| { panic!( "Error trying to read {} as an examples directory: {}", examples_dir, err ); }); for entry in entries { let entry = entry.unwrap(); if entry.file_type().unwrap().is_dir() { let example_dir_name = entry.file_name().into_string().unwrap(); // We test benchmarks separately if example_dir_name != "benchmarks" { all_examples.remove(example_dir_name.as_str()).unwrap_or_else(|| { panic!("The example directory {}/{} does not have any corresponding tests in cli_run. Please add one, so if it ever stops working, we'll know about it right away!", examples_dir, example_dir_name); }); } } } assert_eq!(all_examples, &mut MutMap::default()); } #[cfg(not(debug_assertions))] fn check_for_benchmarks(benchmarks_dir: &str, all_benchmarks: &mut MutMap<&str, Example<'_>>) { use std::ffi::OsStr; use std::fs::File; use std::io::Read; let entries = std::fs::read_dir(benchmarks_dir).unwrap_or_else(|err| { panic!( "Error trying to read {} as a benchmark directory: {}", benchmarks_dir, err ); }); for entry in entries { let entry = entry.unwrap(); let path = entry.path(); if let Some("roc") = path.extension().and_then(OsStr::to_str) { let benchmark_file_name = entry.file_name().into_string().unwrap(); // Verify that this is an app module by reading the first 3 // bytes of the file. let buf: &mut [u8] = &mut [0, 0, 0]; let mut file = File::open(path).unwrap(); file.read_exact(buf).unwrap(); // Only app modules in this directory are considered benchmarks. if "app".as_bytes() == buf { all_benchmarks.remove(benchmark_file_name.as_str()).unwrap_or_else(|| { panic!("The benchmark {}/{} does not have any corresponding tests in cli_run. Please add one, so if it ever stops working, we'll know about it right away!", benchmarks_dir, benchmark_file_name); }); } } } assert_eq!(all_benchmarks, &mut MutMap::default()); } #[test] #[serial(multi_dep_str)] fn run_multi_dep_str_unoptimized() { check_output_with_stdin( &fixture_file("multi-dep-str", "Main.roc"), &[], "multi-dep-str", &[], "I am Dep2.str2\n", true, ); } #[test] #[serial(multi_dep_str)] fn run_multi_dep_str_optimized() { check_output_with_stdin( &fixture_file("multi-dep-str", "Main.roc"), &[], "multi-dep-str", &["--optimize"], "I am Dep2.str2\n", true, ); } #[test] #[serial(multi_dep_thunk)] fn run_multi_dep_thunk_unoptimized() { check_output_with_stdin( &fixture_file("multi-dep-thunk", "Main.roc"), &[], "multi-dep-thunk", &[], "I am Dep2.value2\n", true, ); } #[test] #[serial(multi_dep_thunk)] fn run_multi_dep_thunk_optimized() { check_output_with_stdin( &fixture_file("multi-dep-thunk", "Main.roc"), &[], "multi-dep-thunk", &["--optimize"], "I am Dep2.value2\n", true, ); } }