fix(tests): re-emit a failing test's stderr (#254)

crates/macros/src/test.rs

@@ -26,7 +26,7 @@ pub fn test(attrs: TokenStream, item: TokenStream) -> TokenStream {
         quote!()
     } else {
         quote! {
-            -> ::core::result::Result<(), ::std::string::String>
+            -> ::core::result::Result<(), impl ::core::fmt::Debug>
         }
     };
 

src/tests/terminator.rs

@@ -1,4 +1,4 @@
-use core::fmt::Display;
+use core::fmt;
 use std::panic::PanicHookInfo;
 use std::sync::{Arc, OnceLock};
 
@@ -36,8 +36,8 @@ pub enum TestFailure<'a, E> {
 /// `terminate`, the test will run forever.
 #[cfg_attr(docsrs, doc(cfg(feature = "test-terminator")))]
 pub struct TestTerminator {
-    pub(super) lock: Arc<OnceLock<Result<(), super::test_macro::Failure>>>,
     pub(super) handle: crate::libuv::AsyncHandle,
+    pub(super) result: Arc<OnceLock<super::test_macro::TestResult>>,
 }
 
 impl TestTerminator {
@@ -45,8 +45,11 @@ impl TestTerminator {
     ///
     /// Note that this will have no effect if [`terminate`](Self::terminate)
     /// has already been called.
-    pub fn terminate<E: Display>(&self, res: Result<(), TestFailure<'_, E>>) {
-        if let Ok(()) = self.lock.set(res.map_err(Into::into)) {
+    pub fn terminate<E: fmt::Debug>(
+        &self,
+        result: Result<(), TestFailure<'_, E>>,
+    ) {
+        if let Ok(()) = self.result.set(result.into()) {
             self.handle.send().unwrap();
         }
     }

src/tests/test_macro.rs

@@ -1,14 +1,17 @@
 //! Functions called by the code generated by `#[nvim_oxi::test].`
 
+use core::{fmt, str};
 use std::any::Any;
 use std::env;
-use std::fmt::{Debug, Display};
+use std::io::{self, Write};
 use std::panic::{self, Location, UnwindSafe};
-use std::process::Command;
+use std::process::{Command, Output};
 use std::str::FromStr;
 use std::sync::{Arc, OnceLock};
 use std::thread;
 
+use cargo_metadata::camino::Utf8PathBuf;
+
 use crate::IntoResult;
 
 /// The body of the `#[nvim_oxi::plugin]` generated by the `#[nvim_oxi::test]`
@@ -17,25 +20,21 @@ pub fn plugin_body<F, R>(test_body: F)
 where
     F: FnOnce() -> R + UnwindSafe,
     R: IntoResult<()>,
-    R::Error: Display,
+    R::Error: fmt::Debug,
 {
     let panic_info: Arc<OnceLock<PanicInfo>> = Arc::default();
 
-    {
+    panic::set_hook({
         let panic_info = panic_info.clone();
-
-        panic::set_hook(Box::new(move |info| {
+        Box::new(move |info| {
             let _ = panic_info.set(info.into());
-        }));
-    }
+        })
+    });
 
-    let result = match panic::catch_unwind(|| test_body().into_result()) {
-        Ok(Ok(())) => Ok(()),
-        Ok(Err(err)) => Err(Failure::Error(err.to_string())),
-        Err(_) => Err(Failure::Panic(panic_info.get().unwrap().clone())),
-    };
+    let result = panic::catch_unwind(|| test_body().into_result())
+        .map_err(|_| panic_info.get().unwrap().clone());
 
-    exit(result);
+    exit(&result.into());
 }
 
 /// The body of the `#[nvim_oxi::plugin]` generated by the `#[nvim_oxi::test]`
@@ -46,20 +45,24 @@ pub fn plugin_body_with_terminator<F>(test_body: F)
 where
     F: FnOnce(super::terminator::TestTerminator),
 {
-    let lock = Arc::new(OnceLock::<Result<(), Failure>>::new());
+    let result = Arc::new(OnceLock::<TestResult>::new());
 
-    let handle = {
-        let lock = lock.clone();
-
-        crate::libuv::AsyncHandle::new(move || {
-            let result = lock.get().unwrap().clone();
-            crate::schedule(move |()| exit(result));
-            Ok::<_, std::convert::Infallible>(())
+    let handle =
+        crate::libuv::AsyncHandle::new({
+            let result = result.clone();
+            move || {
+                let result = result.clone();
+                crate::schedule(move |()| {
+                    exit(result.get().expect(
+                        "AsyncHandle triggered before setting TestResult",
+                    ))
+                });
+                Ok::<_, std::convert::Infallible>(())
+            }
         })
-    }
-    .unwrap();
+        .unwrap();
 
-    test_body(super::terminator::TestTerminator { lock, handle });
+    test_body(super::terminator::TestTerminator { handle, result });
 }
 
 /// The body of the `#[test]` generated by the `#[nvim_oxi::test]` macro.
@@ -67,72 +70,228 @@ pub fn test_body(
     manifest_path: &str,
     plugin_name: &str,
     extra_cmd: Option<&str>,
-) -> Result<(), String> {
-    panic::set_hook(Box::new(move |info| {
-        let mut info = info
-            .payload()
-            .downcast_ref::<PanicInfo>()
-            .cloned()
-            .unwrap_or_else(|| info.into());
-
-        if let Some(thread) = thread::current().name() {
-            if !thread.is_empty() {
-                info.thread = thread.to_owned();
-            }
-        }
-
-        eprintln!("{info}");
-    }));
-
-    let output = run_nvim_command(manifest_path, plugin_name, extra_cmd)?
-        .output()
-        .map_err(|err| err.to_string())?;
-
-    let stdout = String::from_utf8_lossy(&output.stdout);
-    let stdout = stdout.trim();
+) -> Result<(), impl fmt::Debug> {
+    let Output { status, stdout, mut stderr } =
+        run_nvim_command(manifest_path, plugin_name, extra_cmd)?
+            .output()
+            .map_err(ExpandedTestError::NeovimProcessFailed)?;
 
+    // Re-emit stdout exactly as received.
     if !stdout.is_empty() {
-        println!("{stdout}");
+        print!("{}", String::from_utf8_lossy(&stdout));
     }
 
-    if output.status.success() {
-        return Ok(());
-    }
-
-    let stderr = String::from_utf8_lossy(&output.stderr);
-    let stderr = stderr.trim();
-
-    if stderr.is_empty() {
-        let msg = output
-            .status
+    // Extract the test result from stderr.
+    let Some(test_result) = TestResult::extract_from_stderr(&mut stderr)
+    else {
+        assert!(!status.success());
+        return Err(status
             .code()
-            .map(|i| format!("Neovim exited with non-zero exit code: {i}"))
-            .unwrap_or_else(|| String::from("Neovim segfaulted"));
-
-        return Err(msg);
-    }
-
-    let Ok(failure) = Failure::from_str(stderr) else {
-        return Err(stderr.to_owned());
+            .map(ExpandedTestError::NeovimExitedWithCode)
+            .unwrap_or(ExpandedTestError::NeovimSegfaulted));
     };
 
-    match failure {
-        Failure::Error(err) => Err(err),
-        Failure::Panic(info) => panic::panic_any(info),
+    // Re-emit the rest of stderr.
+    if !stderr.is_empty() {
+        eprint!("{}", String::from_utf8_lossy(&stderr));
+    }
+
+    match test_result {
+        TestResult::Passed => {
+            assert!(status.success());
+            Ok(())
+        },
+
+        TestResult::Errored(error_msg) => {
+            Err(ExpandedTestError::TestErrored(error_msg))
+        },
+
+        TestResult::Panicked(panic_info) => {
+            panic::set_hook(Box::new(move |info| {
+                let mut info = info
+                    .payload()
+                    .downcast_ref::<PanicInfo>()
+                    .cloned()
+                    .unwrap_or_else(|| info.into());
+
+                if let Some(thread) = thread::current().name() {
+                    if !thread.is_empty() {
+                        info.thread = thread.to_owned();
+                    }
+                }
+
+                eprintln!("\n{info}");
+            }));
+
+            panic::panic_any(panic_info)
+        },
     }
 }
 
-fn exit(result: Result<(), Failure>) {
+#[doc(hidden)]
+pub enum ExpandedTestError {
+    CouldntReadManifest(super::build::BuildError),
+    CouldntReadProfileEnvVar(env::VarError),
+    LibraryNotFound(Utf8PathBuf),
+    NeovimExitedWithCode(i32),
+    NeovimProcessFailed(io::Error),
+    NeovimSegfaulted,
+    TestErrored(String),
+}
+
+pub(super) enum TestResult {
+    Passed,
+    Errored(String),
+    Panicked(PanicInfo),
+}
+
+#[derive(Clone)]
+pub(crate) struct PanicInfo {
+    msg: String,
+    thread: String,
+    file: Option<String>,
+    line: Option<u32>,
+    column: Option<u32>,
+}
+
+impl TestResult {
+    const FENCE_START: &str = "__NVIM_OXI_TEST_RESULT_START__";
+    const FENCE_END: &str = "__NVIM_OXI_TEST_RESULT_END__";
+
+    const PASSED_PREFIX: &str = "passed";
+    const ERRORED_PREFIX: &str = "errored";
+    const PANICKED_PREFIX: &str = "panicked";
+
+    fn embed_in_stderr(&self, stderr: &mut io::Stderr) {
+        write!(
+            stderr,
+            "{fence_start}{self}{fence_end}",
+            fence_start = Self::FENCE_START,
+            fence_end = Self::FENCE_END,
+        )
+        .expect("couldn't write TestResult to stderr");
+    }
+
+    fn extract_from_stderr(stderr: &mut Vec<u8>) -> Option<Self> {
+        let fence_start = stderr
+            .windows(Self::FENCE_START.len())
+            .position(|window| window == Self::FENCE_START.as_bytes())?;
+
+        let fence_end = fence_start
+            + stderr[fence_start..]
+                .windows(Self::FENCE_END.len())
+                .position(|window| window == Self::FENCE_END.as_bytes())?;
+
+        let this = str::from_utf8(
+            &stderr[fence_start + Self::FENCE_START.len()..fence_end],
+        )
+        .ok()?
+        .parse()
+        .ok()?;
+
+        stderr.drain(fence_start..fence_end + Self::FENCE_END.len());
+
+        Some(this)
+    }
+}
+
+impl fmt::Debug for ExpandedTestError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::CouldntReadManifest(err) => {
+                write!(f, "couldn't read manifest: {err}")
+            },
+            Self::CouldntReadProfileEnvVar(err) => {
+                write!(f, "couldn't read profile env var: {err}")
+            },
+            Self::LibraryNotFound(path) => {
+                write!(
+                    f,
+                    "couldn't find library at '{path}'. Did you forget to \
+                     use the build script?"
+                )
+            },
+            Self::NeovimExitedWithCode(code) => {
+                write!(f, "Neovim exited with code {code}")
+            },
+            Self::NeovimProcessFailed(err) => {
+                write!(f, "Neovim process failed: {err}")
+            },
+            Self::NeovimSegfaulted => write!(f, "Neovim segfaulted"),
+            Self::TestErrored(err) => write!(f, "{err}"),
+        }
+    }
+}
+
+impl fmt::Display for TestResult {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Passed => write!(f, "{}", Self::PASSED_PREFIX),
+            Self::Errored(msg) => write!(f, "{}{msg}", Self::ERRORED_PREFIX),
+            Self::Panicked(info) => {
+                write!(f, "{}{info:?}", Self::PANICKED_PREFIX)
+            },
+        }
+    }
+}
+
+impl FromStr for TestResult {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if let Some(empty) = s.strip_prefix(Self::PASSED_PREFIX) {
+            empty.is_empty().then_some(Self::Passed).ok_or(())
+        } else if let Some(info) = s.strip_prefix(Self::PANICKED_PREFIX) {
+            info.parse().map(Self::Panicked)
+        } else if let Some(msg) = s.strip_prefix(Self::ERRORED_PREFIX) {
+            Ok(Self::Errored(msg.to_owned()))
+        } else {
+            Err(())
+        }
+    }
+}
+
+impl<T: fmt::Debug> From<Result<Result<(), T>, PanicInfo>> for TestResult {
+    fn from(result: Result<Result<(), T>, PanicInfo>) -> Self {
+        match result {
+            Ok(Ok(())) => Self::Passed,
+            Ok(Err(err)) => Self::Errored(format!("{:?}", err)),
+            Err(panic_info) => Self::Panicked(panic_info),
+        }
+    }
+}
+
+#[cfg(feature = "test-terminator")]
+impl<T: fmt::Debug> From<Result<(), super::terminator::TestFailure<'_, T>>>
+    for TestResult
+{
+    fn from(
+        result: Result<(), super::terminator::TestFailure<'_, T>>,
+    ) -> Self {
+        match result {
+            Ok(()) => Self::Passed,
+            Err(super::terminator::TestFailure::Error(err)) => {
+                Self::Errored(format!("{:?}", err))
+            },
+            Err(super::terminator::TestFailure::Panic(infos)) => {
+                Self::Panicked(infos.into())
+            },
+        }
+    }
+}
+
+fn exit(result: &TestResult) {
     let exec = |cmd: &str| {
         let opts = crate::api::opts::ExecOpts::builder().output(false).build();
         crate::api::exec2(cmd, &opts).unwrap();
     };
 
-    if let Err(failure) = result {
-        eprintln!("{failure}");
-        exec("cquit 1");
-    } else {
+    result.embed_in_stderr(&mut io::stderr());
+
+    if matches!(result, TestResult::Passed) {
         exec("qall!");
+    } else {
+        exec("cquit 1");
     }
 }
 
@@ -140,20 +299,17 @@ fn run_nvim_command(
     manifest_path: &str,
     plugin_name: &str,
     extra_cmd: Option<&str>,
-) -> Result<Command, String> {
+) -> Result<Command, ExpandedTestError> {
     let manifest = super::build::CargoManifest::from_path(manifest_path)
-        .map_err(|err| err.to_string())?;
+        .map_err(ExpandedTestError::CouldntReadManifest)?;
 
-    let profile =
-        env::var(manifest.profile_env()).map_err(|err| err.to_string())?;
+    let profile = env::var(manifest.profile_env())
+        .map_err(ExpandedTestError::CouldntReadProfileEnvVar)?;
 
     let library_path = manifest.library_path(&profile);
 
     if !library_path.exists() {
-        return Err(format!(
-            "couldn't find library at '{library_path}'. Did you forget to \
-             use the build script?",
-        ));
+        return Err(ExpandedTestError::LibraryNotFound(library_path));
     }
 
     let load_library = format!(
@@ -173,16 +329,25 @@ fn run_nvim_command(
     Ok(command)
 }
 
-#[derive(Clone)]
-pub(super) struct PanicInfo {
-    msg: String,
-    thread: String,
-    file: Option<String>,
-    line: Option<u32>,
-    column: Option<u32>,
+impl fmt::Display for PanicInfo {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "thread '{}' panicked", self.thread)?;
+
+        if let Some(file) = &self.file {
+            write!(f, " at {file}")?;
+
+            if let (Some(line), Some(col)) = (self.line, self.column) {
+                write!(f, ":{line}:{col}")?;
+            }
+        }
+
+        write!(f, ":\n{}", self.msg)?;
+
+        Ok(())
+    }
 }
 
-impl Debug for PanicInfo {
+impl fmt::Debug for PanicInfo {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "panic:{}", self.msg)?;
 
@@ -204,24 +369,6 @@ impl Debug for PanicInfo {
     }
 }
 
-impl Display for PanicInfo {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "thread '{}' panicked", self.thread)?;
-
-        if let Some(file) = &self.file {
-            write!(f, " at {file}")?;
-
-            if let (Some(line), Some(col)) = (self.line, self.column) {
-                write!(f, ":{line}:{col}")?;
-            }
-        }
-
-        write!(f, ":\n{}", self.msg)?;
-
-        Ok(())
-    }
-}
-
 impl FromStr for PanicInfo {
     type Err = ();
 
@@ -282,48 +429,8 @@ impl From<&panic::PanicHookInfo<'_>> for PanicInfo {
     }
 }
 
-#[derive(Clone)]
-pub(super) enum Failure {
-    Error(String),
-    Panic(PanicInfo),
-}
-
-impl Display for Failure {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Failure::Error(err) => write!(f, "error:{err}"),
-            Failure::Panic(info) => write!(f, "{info:?}"),
-        }
-    }
-}
-
-impl FromStr for Failure {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        match s.split_once("error:") {
-            Some((_, msg)) => Ok(Failure::Error(msg.trim().to_owned())),
-            None => PanicInfo::from_str(s).map(Self::Panic),
-        }
-    }
-}
-
-#[cfg(feature = "test-terminator")]
-impl<E: Display> From<super::terminator::TestFailure<'_, E>> for Failure {
-    fn from(err: super::terminator::TestFailure<'_, E>) -> Self {
-        match err {
-            super::terminator::TestFailure::Error(err) => {
-                Self::Error(err.to_string())
-            },
-            super::terminator::TestFailure::Panic(info) => {
-                Self::Panic(info.into())
-            },
-        }
-    }
-}
-
-fn downcast_display<T: Any + Display>(
+fn downcast_display<T: Any + fmt::Display>(
     value: &dyn Any,
-) -> Option<&dyn Display> {
-    value.downcast_ref::<T>().map(|msg| msg as &dyn Display)
+) -> Option<&dyn fmt::Display> {
+    value.downcast_ref::<T>().map(|msg| msg as &dyn fmt::Display)
 }