diff --git a/cli/src/main.rs b/cli/src/main.rs
index db44690c1a..88d3b1eae0 100644
--- a/cli/src/main.rs
+++ b/cli/src/main.rs
@@ -21,12 +21,13 @@ use roc_mono::expr::{Expr, Procs};
 use roc_mono::layout::Layout;
 use std::time::SystemTime;
 
-use clap::{App, AppSettings, Arg, ArgMatches};
+use clap::{App, Arg, ArgMatches};
 use inkwell::targets::{
     CodeModel, FileType, InitializationConfig, RelocMode, Target, TargetTriple,
 };
-use std::io;
+use std::io::{self, ErrorKind};
 use std::path::{Path, PathBuf};
+use std::process;
 use target_lexicon::{Architecture, OperatingSystem, Triple, Vendor};
 use tokio::process::Command;
 use tokio::runtime::Builder;
@@ -41,7 +42,6 @@ pub static FLAG_ROC_FILE: &str = "ROC_FILE";
 pub fn build_app<'a>() -> App<'a> {
     App::new("roc")
         .version(crate_version!())
-        .setting(AppSettings::AllowNegativeNumbers)
         .arg(
             Arg::with_name(FLAG_ROC_FILE)
                 .help("The .roc file to compile and run")
@@ -58,7 +58,6 @@ pub fn build_app<'a>() -> App<'a> {
 /// Run the CLI. This is separate from main() so that tests can call it directly.
 pub fn run(matches: ArgMatches) -> io::Result<()> {
     let filename = matches.value_of(FLAG_ROC_FILE).unwrap();
-
     let opt_level = if matches.is_present(FLAG_OPTIMIZE) {
         OptLevel::Optimize
     } else {
@@ -76,7 +75,24 @@ pub fn run(matches: ArgMatches) -> io::Result<()> {
         .expect("Error spawning initial compiler thread."); // TODO make this error nicer.
 
     // Spawn the root task
-    let loaded = rt.block_on(load_file(src_dir, path.canonicalize().unwrap(), opt_level));
+    let path = path.canonicalize().unwrap_or_else(|err| {
+        use ErrorKind::*;
+
+        match err.kind() {
+            NotFound => {
+                match path.to_str() {
+                    Some(path_str) => println!("File not found: {}", path_str),
+                    None => println!("Malformed file path : {:?}", path),
+                }
+
+                process::exit(1);
+            }
+            _ => {
+                todo!("TODO Gracefully handle opening {:?} - {:?}", path, err);
+            }
+        }
+    });
+    let loaded = rt.block_on(load_file(src_dir, path, opt_level));
 
     loaded.expect("TODO gracefully handle LoadingProblem");
 
diff --git a/cli/tests/cli_run.rs b/cli/tests/cli_run.rs
new file mode 100644
index 0000000000..74e9531f0c
--- /dev/null
+++ b/cli/tests/cli_run.rs
@@ -0,0 +1,105 @@
+#[macro_use]
+extern crate pretty_assertions;
+
+extern crate bumpalo;
+extern crate inlinable_string;
+extern crate roc_collections;
+extern crate roc_load;
+extern crate roc_module;
+
+#[cfg(test)]
+mod cli_run {
+    use std::env;
+    use std::path::PathBuf;
+    use std::process::{Command, ExitStatus};
+
+    // HELPERS
+
+    pub struct Out {
+        pub stdout: String,
+        pub stderr: String,
+        pub status: ExitStatus,
+    }
+
+    pub fn path_to_roc_binary() -> PathBuf {
+        // Adapted from https://github.com/volta-cli/volta/blob/cefdf7436a15af3ce3a38b8fe53bb0cfdb37d3dd/tests/acceptance/support/sandbox.rs#L680 - BSD-2-Clause licensed
+        let mut path = env::var_os("CARGO_BIN_PATH")
+            .map(PathBuf::from)
+            .or_else(|| {
+                env::current_exe().ok().map(|mut path| {
+                    path.pop();
+                    if path.ends_with("deps") { path.pop();
+                    }
+                    path
+                })
+            })
+            .unwrap_or_else(|| panic!("CARGO_BIN_PATH wasn't set, and couldn't be inferred from context. Can't run CLI tests."));
+
+        path.push("roc");
+
+        path
+    }
+
+    pub fn run_roc(args: &[&str]) -> Out {
+        let mut cmd = Command::new(path_to_roc_binary());
+
+        for arg in args {
+            cmd.arg(arg);
+        }
+
+        let output = cmd
+            .output()
+            .expect("failed to execute compiled `roc` binary in CLI test");
+
+        Out {
+            stdout: String::from_utf8(output.stdout).unwrap(),
+            stderr: String::from_utf8(output.stderr).unwrap(),
+            status: output.status,
+        }
+    }
+
+    pub fn example_dir(dir_name: &str) -> PathBuf {
+        let mut path = env::current_exe().ok().unwrap();
+
+        // Get rid of the filename in target/debug/deps/cli_run-99c65e4e9a1fbd06
+        path.pop();
+
+        // If we're in deps/ get rid of deps/ in target/debug/deps/
+        if path.ends_with("deps") {
+            path.pop();
+        }
+
+        // Get rid of target/debug/ so we're back at the project root
+        path.pop();
+        path.pop();
+
+        // Descend into examples/{dir_name}
+        path.push("examples");
+        path.push(dir_name);
+
+        path
+    }
+
+    pub fn example_file(dir_name: &str, file_name: &str) -> PathBuf {
+        let mut path = example_dir(dir_name);
+
+        path.push(file_name);
+
+        path
+    }
+
+    // TESTS
+
+    #[test]
+    fn run_hello_world() {
+        let out = run_roc(&[example_file("hello-world", "Hello.roc").to_str().unwrap()]);
+
+        assert_eq!(&out.stderr, "");
+
+        // TODO make separate `roc build` and `roc run` commands, and here do
+        // `roc build` followed by manually executing the compiled `app` binary
+        // and doing an `assert_eq!` on the entire stdout of that compiled `app` binary
+        assert!(&out.stdout.ends_with("Hello, World!\n"));
+        assert!(out.status.success());
+    }
+}