feat: add test framework with coverage support (#1518)

* feat: test framework with coverage support

* feat: clean up

* fix: dangling else block

* feat: implement show set

* feat: implement show replace content

* feat: compare framework

* feat: preserve extension

* feat: exit 1 if failed testing

* docs: update docs about it

* docs: wording

* docs: wording

* docs: wording

crates/tinymist-debug/src/cov.rs

@@ -154,7 +154,7 @@ pub fn __cov_pc(span: Span, pc: i64) {
 pub enum Kind {
     OpenBrace,
     CloseBrace,
-    Functor,
+    Show,
 }
 
 #[derive(Default)]
@@ -221,16 +221,27 @@ impl InstrumentWorker {
                 }
                 ast::Expr::Show(show_rule) => {
                     let transform = show_rule.transform().to_untyped().span();
+                    let is_set = matches!(show_rule.transform(), ast::Expr::Set(..));
 
                     for child in node.children() {
                         if transform == child.span() {
-                            self.instrument_functor(child);
+                            if is_set {
+                                self.instrument_show_set(child);
+                            } else {
+                                self.instrument_show_transform(child);
+                            }
                         } else {
                             self.visit_node(child);
                         }
                     }
                     return;
                 }
+                ast::Expr::Contextual(body) => {
+                    self.instrumented.push_str("context (");
+                    self.visit_node(body.body().to_untyped());
+                    self.instrumented.push(')');
+                    return;
+                }
                 ast::Expr::Text(..)
                 | ast::Expr::Space(..)
                 | ast::Expr::Linebreak(..)
@@ -278,7 +289,6 @@ impl InstrumentWorker {
                 | ast::Expr::Let(..)
                 | ast::Expr::DestructAssign(..)
                 | ast::Expr::Set(..)
-                | ast::Expr::Contextual(..)
                 | ast::Expr::Import(..)
                 | ast::Expr::Include(..)
                 | ast::Expr::Break(..)
@@ -328,16 +338,24 @@ impl InstrumentWorker {
         self.visit_node_fallback(child);
         self.instrumented.push('\n');
         self.make_cov(last, Kind::CloseBrace);
-        self.instrumented.push_str("}\n");
+        self.instrumented.push('}');
     }
 
-    fn instrument_functor(&mut self, child: &SyntaxNode) {
+    fn instrument_show_set(&mut self, child: &SyntaxNode) {
-        self.instrumented.push_str("{\nlet __cov_functor = ");
+        self.instrumented.push_str("__it => {");
+        self.make_cov(child.span(), Kind::Show);
+        self.visit_node(child);
+        self.instrumented.push_str("\n__it; }\n");
+    }
+
+    fn instrument_show_transform(&mut self, child: &SyntaxNode) {
+        self.instrumented.push_str("{\nlet __cov_show_body = ");
         let s = child.span();
-        self.visit_node_fallback(child);
+        self.visit_node(child);
         self.instrumented.push_str("\n__it => {");
-        self.make_cov(s, Kind::Functor);
+        self.make_cov(s, Kind::Show);
-        self.instrumented.push_str("__cov_functor(__it); } }\n");
+        self.instrumented
+            .push_str("if type(__cov_show_body) == function { __cov_show_body(__it); } else { __cov_show_body } } }\n");
     }
 }
 
@@ -354,7 +372,7 @@ mod tests {
     #[test]
     fn test_physica_vector() {
         let instrumented = instr(include_str!("fixtures/instr_coverage/physica_vector.typ"));
-        insta::assert_snapshot!(instrumented, @r#"
+        insta::assert_snapshot!(instrumented, @r###"
         // A show rule, should be used like:
         //   #show: super-plus-as-dagger
         //   U^+U = U U^+ = I
@@ -367,7 +385,7 @@ mod tests {
         __cov_pc(0);
         {
           show math.attach: {
-        let __cov_functor = elem => {
+        let __cov_show_body = elem => {
         __cov_pc(1);
         {
             if __eligible(elem.base) and elem.at("t", default: none) == [+] {
@@ -376,28 +394,25 @@ mod tests {
               $attach(elem.base, t: dagger, b: elem.at("b", default: #none))$
             }
         __cov_pc(3);
-        }
+        } else {
-         else {
         __cov_pc(4);
         {
               elem
             }
         __cov_pc(5);
         }
-
           }
         __cov_pc(6);
         }
-
         __it => {__cov_pc(7);
-        __cov_functor(__it); } }
+        if type(__cov_show_body) == function { __cov_show_body(__it); } else { __cov_show_body } } }
 
 
           document
         }
         __cov_pc(8);
         }
-        "#);
+        "###);
     }
 
     #[test]
@@ -413,29 +428,99 @@ mod tests {
         insta::assert_snapshot!(new.text(), @"#let a = 1;");
     }
 
+    #[test]
+    fn test_instrument_coverage_show_content() {
+        let source = Source::detached("#show math.equation: context it => it");
+        let (new, _meta) = instrument_coverage(source).unwrap();
+        insta::assert_snapshot!(new.text(), @r###"
+        #show math.equation: {
+        let __cov_show_body = context (it => {
+        __cov_pc(0);
+        it
+        __cov_pc(1);
+        })
+        __it => {__cov_pc(2);
+        if type(__cov_show_body) == function { __cov_show_body(__it); } else { __cov_show_body } } }
+        "###);
+    }
+
+    #[test]
+    fn test_instrument_inline_block() {
+        let source = Source::detached("#let main-size = {1} + 2 + {3}");
+        let (new, _meta) = instrument_coverage(source).unwrap();
+        insta::assert_snapshot!(new.text(), @r###"
+        #let main-size = {
+        __cov_pc(0);
+        {1}
+        __cov_pc(1);
+        } + 2 + {
+        __cov_pc(2);
+        {3}
+        __cov_pc(3);
+        }
+        "###);
+    }
+
+    #[test]
+    fn test_instrument_if() {
+        let source = Source::detached(
+            "#let main-size = if is-web-target {
+  16pt
+} else {
+  10.5pt
+}",
+        );
+        let (new, _meta) = instrument_coverage(source).unwrap();
+        insta::assert_snapshot!(new.text(), @r###"
+        #let main-size = if is-web-target {
+        __cov_pc(0);
+        {
+          16pt
+        }
+        __cov_pc(1);
+        } else {
+        __cov_pc(2);
+        {
+          10.5pt
+        }
+        __cov_pc(3);
+        }
+        "###);
+    }
+
     #[test]
     fn test_instrument_coverage_nested() {
         let source = Source::detached("#let a = {1};");
         let (new, _meta) = instrument_coverage(source).unwrap();
-        insta::assert_snapshot!(new.text(), @r"
+        insta::assert_snapshot!(new.text(), @r###"
         #let a = {
         __cov_pc(0);
         {1}
         __cov_pc(1);
-        }
+        };
-        ;
+        "###);
-        ");
     }
 
     #[test]
     fn test_instrument_coverage_functor() {
         let source = Source::detached("#show: main");
         let (new, _meta) = instrument_coverage(source).unwrap();
-        insta::assert_snapshot!(new.text(), @r"
+        insta::assert_snapshot!(new.text(), @r###"
         #show: {
-        let __cov_functor = main
+        let __cov_show_body = main
         __it => {__cov_pc(0);
-        __cov_functor(__it); } }
+        if type(__cov_show_body) == function { __cov_show_body(__it); } else { __cov_show_body } } }
-        ");
+        "###);
+    }
+
+    #[test]
+    fn test_instrument_coverage_set() {
+        let source = Source::detached("#show raw: set text(12pt)");
+        let (new, _meta) = instrument_coverage(source).unwrap();
+        insta::assert_snapshot!(new.text(), @r###"
+        #show raw: __it => {__cov_pc(0);
+        set text(12pt)
+        __it; }
+        "###);
     }
 }

crates/tinymist-debug/src/lib.rs

@@ -43,10 +43,10 @@ pub fn collect_coverage<D: typst::Document, F: CompilerFeat>(
 }
 
 /// Collects the coverage with a callback.
-pub fn with_cov<F: CompilerFeat>(
+pub fn with_cov<F: CompilerFeat, T>(
     base: &CompilerWorld<F>,
-    mut f: impl FnMut(&InstrumentWorld<F, CovInstr>) -> Result<()>,
+    mut f: impl FnMut(&InstrumentWorld<F, CovInstr>) -> Result<T>,
-) -> (Result<CoverageResult>, Result<()>) {
+) -> (Result<CoverageResult>, Result<T>) {
     let instr = InstrumentWorld {
         base,
         library: instrument_library(&base.library),

crates/tinymist-project/src/args.rs

@@ -157,14 +157,13 @@ pub struct TaskCompileArgs {
     #[arg(long = "pages", value_delimiter = ',')]
     pub pages: Option<Vec<Pages>>,
 
-    /// One (or multiple comma-separated) PDF standards that Typst will enforce
+    /// The argument to export to PDF.
-    /// conformance with.
+    #[clap(flatten)]
-    #[arg(long = "pdf-standard", value_delimiter = ',')]
+    pub pdf: PdfExportArgs,
-    pub pdf_standard: Vec<PdfStandard>,
 
-    /// The PPI (pixels per inch) to use for PNG export.
+    /// The argument to export to PNG.
-    #[arg(long = "ppi", default_value_t = 144.0)]
+    #[clap(flatten)]
-    pub ppi: f32,
+    pub png: PngExportArgs,
 
     /// The output format.
     #[clap(skip)]
@@ -215,12 +214,12 @@ impl TaskCompileArgs {
         let config = match output_format {
             OutputFormat::Pdf => ProjectTask::ExportPdf(ExportPdfTask {
                 export,
-                pdf_standards: self.pdf_standard.clone(),
+                pdf_standards: self.pdf.pdf_standard.clone(),
                 creation_timestamp: None,
             }),
             OutputFormat::Png => ProjectTask::ExportPng(ExportPngTask {
                 export,
-                ppi: self.ppi.try_into().unwrap(),
+                ppi: self.png.ppi.try_into().unwrap(),
                 fill: None,
             }),
             OutputFormat::Svg => ProjectTask::ExportSvg(ExportSvgTask { export }),
@@ -234,3 +233,20 @@ impl TaskCompileArgs {
         })
     }
 }
+
+/// Declare arguments for exporting a document to PDF.
+#[derive(Debug, Clone, clap::Parser)]
+pub struct PdfExportArgs {
+    /// One (or multiple comma-separated) PDF standards that Typst will enforce
+    /// conformance with.
+    #[arg(long = "pdf-standard", value_delimiter = ',')]
+    pub pdf_standard: Vec<PdfStandard>,
+}
+
+/// Declare arguments for exporting a document to PNG.
+#[derive(Debug, Clone, clap::Parser)]
+pub struct PngExportArgs {
+    /// The PPI (pixels per inch) to use for PNG export.
+    #[arg(long = "ppi", default_value_t = 144.0)]
+    pub ppi: f32,
+}

crates/tinymist-project/src/world.rs

@@ -4,7 +4,8 @@ pub use tinymist_world as base;
 pub use tinymist_world::args::*;
 pub use tinymist_world::config::CompileFontOpts;
 pub use tinymist_world::entry::*;
-pub use tinymist_world::{font, package, vfs};
+pub use tinymist_world::{font, package, system, vfs};
 pub use tinymist_world::{
-    CompilerUniverse, CompilerWorld, EntryOpts, EntryState, RevisingUniverse, TaskInputs,
+    CompilerUniverse, CompilerWorld, DiagnosticFormat, EntryOpts, EntryState, RevisingUniverse,
+    TaskInputs,
 };

crates/tinymist-query/src/lib.rs

@@ -12,6 +12,7 @@ pub mod analysis;
 pub mod docs;
 pub mod package;
 pub mod syntax;
+pub mod testing;
 pub mod ty;
 mod upstream;
 

crates/tinymist-query/src/testing/mod.rs

@@ -0,0 +1,229 @@
+//! Extracts test suites from the document.
+
+use ecow::EcoString;
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use tinymist_std::error::prelude::*;
+use tinymist_std::typst::TypstDocument;
+use tinymist_world::vfs::FileId;
+use typst::{
+    foundations::{Func, Label, Module, Selector, Value},
+    introspection::MetadataElem,
+    syntax::Source,
+    utils::PicoStr,
+    World,
+};
+
+use crate::LocalContext;
+
+/// Test suites extracted from the document.
+pub struct TestSuites {
+    /// Files from the current workspace.
+    pub origin_files: Vec<(Source, Module)>,
+    /// Test cases in the current workspace.
+    pub tests: Vec<TestCase>,
+    /// Example documents in the current workspace.
+    pub examples: Vec<Source>,
+}
+impl TestSuites {
+    /// Rechecks the test suites.
+    pub fn recheck(&self, world: &dyn World) -> TestSuites {
+        let tests = self
+            .tests
+            .iter()
+            .filter_map(|test| {
+                let source = world.source(test.location).ok()?;
+                let module = typst_shim::eval::eval_compat(world, &source).ok()?;
+                let symbol = module.scope().get(&test.name)?;
+                let Value::Func(function) = symbol.read() else {
+                    return None;
+                };
+                Some(TestCase {
+                    name: test.name.clone(),
+                    location: test.location,
+                    function: function.clone(),
+                    kind: test.kind,
+                })
+            })
+            .collect();
+
+        let examples = self
+            .examples
+            .iter()
+            .filter_map(|source| world.source(source.id()).ok())
+            .collect();
+
+        TestSuites {
+            origin_files: self.origin_files.clone(),
+            tests,
+            examples,
+        }
+    }
+}
+
+/// Kind of the test case.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TestCaseKind {
+    /// A normal test case.
+    Test,
+    /// A test case that should panic.
+    Panic,
+    /// A benchmark test case.
+    Bench,
+    /// An example test case.
+    Example,
+}
+
+/// A test case.
+pub struct TestCase {
+    /// Name of the test case.
+    pub name: EcoString,
+    /// Location of the test case.
+    pub location: FileId,
+    /// entry of the test case.
+    pub function: Func,
+    /// Kind of the test case.
+    pub kind: TestCaseKind,
+}
+
+/// Extracts the test suites in the document
+pub fn test_suites(ctx: &mut LocalContext, doc: &TypstDocument) -> Result<TestSuites> {
+    let main_id = ctx.world.main();
+    let main_workspace = main_id.package();
+
+    crate::log_debug_ct!(
+        "test workspace: {:?}, files: {:?}",
+        main_workspace,
+        ctx.depended_source_files()
+    );
+    let files = ctx
+        .depended_source_files()
+        .par_iter()
+        .filter(|fid| fid.package() == main_workspace)
+        .map(|fid| {
+            let source = ctx
+                .source_by_id(*fid)
+                .context_ut("failed to get source by id")?;
+            let module = ctx.module_by_id(*fid)?;
+            Ok((source, module))
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    let config = extract_test_configuration(doc)?;
+
+    let mut worker = TestSuitesWorker {
+        files: &files,
+        config,
+        tests: Vec::new(),
+        examples: Vec::new(),
+    };
+
+    worker.discover_tests()?;
+
+    Ok(TestSuites {
+        tests: worker.tests,
+        examples: worker.examples,
+        origin_files: files,
+    })
+}
+
+#[derive(Debug, Clone)]
+struct TestConfig {
+    test_pattern: EcoString,
+    bench_pattern: EcoString,
+    panic_pattern: EcoString,
+    example_pattern: EcoString,
+}
+
+#[derive(Debug, Clone, Default, serde::Deserialize)]
+struct UserTestConfig {
+    test_pattern: Option<EcoString>,
+    bench_pattern: Option<EcoString>,
+    panic_pattern: Option<EcoString>,
+    example_pattern: Option<EcoString>,
+}
+
+fn extract_test_configuration(doc: &TypstDocument) -> Result<TestConfig> {
+    let selector = Label::new(PicoStr::intern("test-config"));
+    let metadata = doc.introspector().query(&Selector::Label(selector));
+    if metadata.len() > 1 {
+        // todo: attach source locations.
+        bail!("multiple test configurations found");
+    }
+
+    let config = if let Some(metadata) = metadata.first() {
+        let metadata = metadata
+            .to_packed::<MetadataElem>()
+            .context("test configuration is not a metadata element")?;
+
+        let value =
+            serde_json::to_value(&metadata.value).context("failed to serialize metadata")?;
+        serde_json::from_value(value).context("failed to deserialize metadata")?
+    } else {
+        UserTestConfig::default()
+    };
+
+    Ok(TestConfig {
+        test_pattern: config.test_pattern.unwrap_or_else(|| "test-".into()),
+        bench_pattern: config.bench_pattern.unwrap_or_else(|| "bench-".into()),
+        panic_pattern: config.panic_pattern.unwrap_or_else(|| "panic-on-".into()),
+        example_pattern: config.example_pattern.unwrap_or_else(|| "example-".into()),
+    })
+}
+
+struct TestSuitesWorker<'a> {
+    files: &'a [(Source, Module)],
+    config: TestConfig,
+    tests: Vec<TestCase>,
+    examples: Vec<Source>,
+}
+
+impl TestSuitesWorker<'_> {
+    fn match_test(&self, name: &str) -> Option<TestCaseKind> {
+        if name.starts_with(self.config.test_pattern.as_str()) {
+            Some(TestCaseKind::Test)
+        } else if name.starts_with(self.config.bench_pattern.as_str()) {
+            Some(TestCaseKind::Bench)
+        } else if name.starts_with(self.config.panic_pattern.as_str()) {
+            Some(TestCaseKind::Panic)
+        } else if name.starts_with(self.config.example_pattern.as_str()) {
+            Some(TestCaseKind::Example)
+        } else {
+            None
+        }
+    }
+
+    fn discover_tests(&mut self) -> Result<()> {
+        for (source, module) in self.files.iter() {
+            let vpath = source.id().vpath().as_rooted_path();
+            let file_name = vpath.file_name().and_then(|s| s.to_str()).unwrap_or("");
+            if file_name.starts_with(self.config.example_pattern.as_str()) {
+                self.examples.push(source.clone());
+                continue;
+            }
+
+            for (name, symbol) in module.scope().iter() {
+                crate::log_debug_ct!("symbol({name:?}): {symbol:?}");
+                let Value::Func(function) = symbol.read() else {
+                    continue;
+                };
+
+                let span = symbol.span();
+                let id = span.id();
+                if Some(source.id()) != id {
+                    continue;
+                }
+
+                if let Some(kind) = self.match_test(name.as_str()) {
+                    self.tests.push(TestCase {
+                        name: name.clone(),
+                        location: source.id(),
+                        function: function.clone(),
+                        kind,
+                    });
+                }
+            }
+        }
+
+        Ok(())
+    }
+}

crates/tinymist/src/args.rs

@@ -3,6 +3,7 @@ use std::path::Path;
 use sync_ls::transport::MirrorArgs;
 use tinymist::project::DocCommands;
 use tinymist::tool::project::{CompileArgs, GenerateScriptArgs, TaskCommands};
+use tinymist::tool::testing::TestArgs;
 use tinymist::{CompileFontArgs, CompileOnceArgs};
 use tinymist_core::LONG_VERSION;
 
@@ -36,6 +37,9 @@ pub enum Commands {
     /// Execute a document and collect coverage
     #[clap(hide(true))] // still in development
     Cov(CompileOnceArgs),
+    /// Test a document and gives summary
+    #[clap(hide(true))] // still in development
+    Test(TestArgs),
     /// Runs compile command like `typst-cli compile`
     Compile(CompileArgs),
     /// Generates build script for compilation

crates/tinymist/src/main.rs

@@ -18,9 +18,8 @@ use sync_ls::{
     internal_error, DapBuilder, DapMessage, LspBuilder, LspClientRoot, LspMessage, LspResult,
     RequestId,
 };
-use tinymist::tool::project::{
+use tinymist::tool::project::{compile_main, generate_script_main, project_main, task_main};
-    compile_main, coverage_main, generate_script_main, project_main, task_main,
+use tinymist::tool::testing::{coverage_main, test_main};
-};
 use tinymist::world::TaskInputs;
 use tinymist::{Config, DapRegularInit, RegularInit, ServerState, SuperInit, UserActionTask};
 use tinymist_core::LONG_VERSION;
@@ -93,6 +92,7 @@ fn main() -> Result<()> {
     match args.command.unwrap_or_default() {
         Commands::Completion(args) => completion(args),
         Commands::Cov(args) => coverage_main(args),
+        Commands::Test(args) => test_main(args),
         Commands::Compile(args) => RUNTIMES.tokio_runtime.block_on(compile_main(args)),
         Commands::GenerateScript(args) => generate_script_main(args),
         Commands::Query(query_cmds) => query_main(query_cmds),

crates/tinymist/src/tool/mod.rs

@@ -2,6 +2,7 @@
 
 pub mod package;
 pub mod project;
+pub mod testing;
 pub mod word_count;
 
 #[cfg(feature = "preview")]

crates/tinymist/src/tool/project.rs

@@ -7,9 +7,9 @@ use std::{
 
 use clap_complete::Shell;
 use reflexo::{path::unix_slash, ImmutPath};
-use reflexo_typst::{diag::print_diagnostics, DiagnosticFormat};
 use tinymist_std::{bail, error::prelude::*};
 
+use crate::world::system::print_diagnostics;
 use crate::{project::*, task::ExportTask};
 
 /// Arguments for project compilation.
@@ -106,41 +106,6 @@ impl LockFileExt for LockFile {
     }
 }
 
-/// Runs project compilation(s)
-pub fn coverage_main(args: CompileOnceArgs) -> Result<()> {
-    // Prepares for the compilation
-    let universe = args.resolve()?;
-    let world = universe.snapshot();
-
-    let result = Ok(()).and_then(|_| -> Result<()> {
-        let res =
-            tinymist_debug::collect_coverage::<tinymist_std::typst::TypstPagedDocument, _>(&world)?;
-        let cov_path = Path::new("target/coverage.json");
-        let res = serde_json::to_string(&res.to_json(&world)).context("coverage")?;
-
-        std::fs::create_dir_all(cov_path.parent().context("parent")?).context("create coverage")?;
-        std::fs::write(cov_path, res).context("write coverage")?;
-
-        Ok(())
-    });
-
-    print_diag_or_error(&world, result)
-}
-
-fn print_diag_or_error(world: &LspWorld, result: Result<()>) -> Result<()> {
-    if let Err(e) = result {
-        if let Some(diagnostics) = e.diagnostics() {
-            print_diagnostics(world, diagnostics.iter(), DiagnosticFormat::Human)
-                .context_ut("print diagnostics")?;
-            bail!("");
-        }
-
-        return Err(e);
-    }
-
-    Ok(())
-}
-
 /// Runs project compilation(s)
 pub async fn compile_main(args: CompileArgs) -> Result<()> {
     // Identifies the input and output

crates/tinymist/src/tool/testing.rs

@@ -0,0 +1,493 @@
+//! Testing utilities
+
+use std::{
+    collections::HashSet,
+    path::Path,
+    sync::{atomic::AtomicBool, Arc},
+};
+
+use comemo::Track;
+use parking_lot::Mutex;
+use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
+use reflexo::ImmutPath;
+use reflexo_typst::{vfs::FileId, Bytes, LazyHash, SourceWorld, TypstDocument, TypstHtmlDocument};
+use tinymist_project::world::{system::print_diagnostics, DiagnosticFormat};
+use tinymist_query::{
+    analysis::Analysis,
+    syntax::{find_source_by_expr, node_ancestors},
+    testing::{TestCaseKind, TestSuites},
+};
+use tinymist_std::{bail, error::prelude::*, typst::TypstPagedDocument};
+use typst::{
+    diag::{FileResult, SourceDiagnostic, SourceResult, Warned},
+    ecow::EcoVec,
+    engine::{Engine, Route, Sink, Traced},
+    foundations::{Context, Datetime, Label, Value},
+    introspection::Introspector,
+    syntax::{ast, LinkedNode, Source, Span},
+    text::{Font, FontBook},
+    utils::PicoStr,
+    Library, World,
+};
+
+use crate::project::*;
+
+/// Runs coverage test on a document
+pub fn coverage_main(args: CompileOnceArgs) -> Result<()> {
+    // Prepares for the compilation
+    let universe = args.resolve()?;
+    let world = universe.snapshot();
+
+    let result = Ok(()).and_then(|_| -> Result<()> {
+        let res = tinymist_debug::collect_coverage::<TypstPagedDocument, _>(&world)?;
+        let cov_path = Path::new("target/coverage.json");
+        let res = serde_json::to_string(&res.to_json(&world)).context("coverage")?;
+
+        std::fs::create_dir_all(cov_path.parent().context("parent")?).context("create coverage")?;
+        std::fs::write(cov_path, res).context("write coverage")?;
+
+        Ok(())
+    });
+
+    print_diag_or_error(&world, result)
+}
+
+/// Testing arguments
+#[derive(Debug, Clone, clap::Parser)]
+pub struct TestArgs {
+    /// The argument to compile once.
+    #[clap(flatten)]
+    pub compile: CompileOnceArgs,
+
+    /// Configuration for testing
+    #[clap(flatten)]
+    pub config: TestConfigArgs,
+}
+
+/// Testing config arguments
+#[derive(Debug, Clone, clap::Parser)]
+pub struct TestConfigArgs {
+    /// Whether to update the reference images.
+    #[clap(long)]
+    pub update: bool,
+
+    /// The argument to export to PNG.
+    #[clap(flatten)]
+    pub png: PngExportArgs,
+}
+
+/// Runs tests on a document
+pub fn test_main(args: TestArgs) -> Result<()> {
+    // Prepares for the compilation
+    let universe = args.compile.resolve()?;
+    let world = universe.snapshot();
+    let root = universe.entry_state().root().map(Ok);
+    let root = root
+        .unwrap_or_else(|| std::env::current_dir().map(|p| p.into()))
+        .context("cannot find root")?;
+
+    let config = TestConfig {
+        root,
+        args: args.config,
+    };
+
+    let result = Ok(()).and_then(|_| -> Result<bool> {
+        let analysis = Analysis::default();
+
+        let mut ctx = analysis.snapshot(world.clone());
+        let doc = typst::compile::<TypstPagedDocument>(&ctx.world).output?;
+
+        let suites =
+            tinymist_query::testing::test_suites(&mut ctx, &TypstDocument::from(Arc::new(doc)))
+                .context("failed to find suites")?;
+        eprintln!(
+            "Found {} tests and {} examples",
+            suites.tests.len(),
+            suites.examples.len()
+        );
+
+        let (cov, result) = tinymist_debug::with_cov(&world, |world| {
+            let suites = suites.recheck(world);
+            let runner = TestRunner::new(config.clone(), &world, &suites);
+            print_diag_or_error(world, runner.run())
+        });
+        let cov = cov?;
+        let cov_path = Path::new("target/coverage.json");
+        let res = serde_json::to_string(&cov.to_json(&world)).context("coverage")?;
+
+        std::fs::create_dir_all(cov_path.parent().context("parent")?).context("create coverage")?;
+        std::fs::write(cov_path, res).context("write coverage")?;
+
+        eprintln!("  Info: Written coverage to {} ...", cov_path.display());
+
+        result
+    });
+
+    let passed = print_diag_or_error(&world, result);
+
+    if matches!(passed, Ok(true)) {
+        eprintln!("  Info: All test cases passed...");
+    } else {
+        eprintln!(" Fatal: some test cases failed...");
+        std::process::exit(1);
+    }
+
+    passed.map(|_| ())
+}
+
+#[derive(Debug, Clone)]
+struct TestConfig {
+    root: ImmutPath,
+    args: TestConfigArgs,
+}
+
+struct TestRunner<'a> {
+    config: TestConfig,
+    world: &'a dyn World,
+    suites: &'a TestSuites,
+    diagnostics: Mutex<Vec<EcoVec<SourceDiagnostic>>>,
+    examples: Mutex<HashSet<String>>,
+    failed: AtomicBool,
+}
+
+impl<'a> TestRunner<'a> {
+    fn new(config: TestConfig, world: &'a dyn World, suites: &'a TestSuites) -> Self {
+        Self {
+            config,
+            world,
+            suites,
+            diagnostics: Mutex::new(Vec::new()),
+            examples: Mutex::new(HashSet::new()),
+            failed: AtomicBool::new(false),
+        }
+    }
+
+    fn mark_failed(&self) {
+        self.failed.store(true, std::sync::atomic::Ordering::SeqCst);
+    }
+
+    fn collect_diag<T>(&self, result: Warned<SourceResult<T>>) -> Option<T> {
+        if !result.warnings.is_empty() {
+            self.diagnostics.lock().push(result.warnings);
+        }
+
+        match result.output {
+            Ok(v) => Some(v),
+            Err(e) => {
+                self.diagnostics.lock().push(e);
+                None
+            }
+        }
+    }
+
+    /// Runs the tests and returns whether all tests passed.
+    fn run(self) -> Result<bool> {
+        rayon::in_place_scope(|s| {
+            s.spawn(|_| {
+                self.suites.tests.par_iter().for_each(|test| {
+                    let name = &test.name;
+                    eprintln!("Running test({name})");
+                    let world = with_main(self.world, test.location);
+                    let introspector = Introspector::default();
+                    let traced = Traced::default();
+                    let route = Route::default();
+                    let mut sink = Sink::default();
+                    let engine = &mut Engine {
+                        routines: &typst::ROUTINES,
+                        world: ((&world) as &dyn World).track(),
+                        introspector: introspector.track(),
+                        traced: traced.track(),
+                        sink: sink.track_mut(),
+                        route,
+                    };
+
+                    let func = &test.function;
+
+                    // Runs the benchmark once.
+                    let mut call_once = move || {
+                        let context = Context::default();
+                        let values = Vec::<Value>::default();
+                        func.call(engine, context.track(), values)
+                    };
+
+                    // Executes the function
+                    match test.kind {
+                        TestCaseKind::Test | TestCaseKind::Bench => {
+                            if let Err(err) = call_once() {
+                                eprintln!(" Failed test({name}): call error {err:?}");
+                                self.mark_failed();
+                            } else {
+                                eprintln!(" Passed test({name})");
+                            }
+                        }
+                        TestCaseKind::Panic => match call_once() {
+                            Ok(..) => {
+                                eprintln!("{name} exited normally, expected panic");
+                            }
+                            Err(err) => {
+                                let has_panic = err.iter().any(|p| p.message.contains("panic"));
+
+                                if !has_panic {
+                                    eprintln!(" Failed test({name}): exited with error, expected panic");
+                                    self.diagnostics.lock().push(err);
+                                    self.mark_failed();
+                                } else {
+                                    eprintln!(" Passed test({name})");
+                                }
+                            }
+                        },
+                        TestCaseKind::Example => {
+                            let example =
+                                get_example_file(&world, name, test.location, func.span());
+                            match example {
+                                Err(err) => {
+                                    eprintln!("cannot find example file in {name}: {err:?}");
+                                    self.mark_failed();
+                                }
+                                Ok(example) => self.run_example(&example),
+                            };
+                        }
+                    }
+                    comemo::evict(30);
+                });
+                self.suites.examples.par_iter().for_each(|test| {
+                    self.run_example(test);
+                    comemo::evict(30);
+                });
+            });
+        });
+
+        {
+            let diagnostics = self.diagnostics.into_inner();
+            if !diagnostics.is_empty() {
+                let diags = diagnostics
+                    .into_iter()
+                    .flat_map(|e| e.into_iter())
+                    .collect::<EcoVec<_>>();
+                Err(diags)?
+            }
+        }
+        Ok(!self.failed.load(std::sync::atomic::Ordering::SeqCst))
+    }
+
+    fn run_example(&self, test: &Source) {
+        let example_path = test.id().vpath().as_rooted_path().with_extension("");
+        let example = example_path
+            .file_name()
+            .and_then(|s| s.to_str())
+            .unwrap_or_default();
+        eprintln!("Running example({example}");
+        if !self.examples.lock().insert(example.to_string()) {
+            eprintln!(" Failed example({example}: duplicate");
+            self.mark_failed();
+            return;
+        }
+        let world = with_main(self.world, test.id());
+        let mut has_err = false;
+        let doc = self.collect_diag(typst::compile::<TypstPagedDocument>(&world));
+        has_err |= doc.is_none();
+        if let Err(err) = self.render_paged(example, doc.as_ref()) {
+            eprintln!("cannot render example({example}, Paged): {err}");
+            has_err = true;
+        }
+
+        if self.can_html(doc.as_ref()) {
+            let doc = self.collect_diag(typst::compile::<TypstHtmlDocument>(&world));
+            has_err |= doc.is_none();
+
+            if let Err(err) = self.render_html(example, doc.as_ref()) {
+                eprintln!("cannot render example({example}, Html): {err}");
+                has_err = true;
+            }
+        }
+
+        if has_err {
+            eprintln!(" Failed example({example}");
+            self.mark_failed();
+        } else {
+            eprintln!(" Passed example({example})");
+        }
+    }
+
+    fn render_paged(&self, example: &str, doc: Option<&TypstPagedDocument>) -> Result<()> {
+        let Some(doc) = doc else {
+            return Ok(());
+        };
+        let pixmap = typst_render::render_merged(
+            doc,
+            self.config.args.png.ppi / 72.0,
+            Default::default(),
+            None,
+        );
+        let output = pixmap.encode_png().context_ut("cannot encode pixmap")?;
+
+        let ref_image = self
+            .config
+            .root
+            .join("refs/png")
+            .join(example)
+            .with_extension("png");
+
+        self.update_example(example, &output, &ref_image, "image")
+    }
+
+    fn render_html(&self, example: &str, doc: Option<&TypstHtmlDocument>) -> Result<()> {
+        let Some(doc) = doc else {
+            return Ok(());
+        };
+        let output = typst_html::html(doc)?.into_bytes();
+
+        let ref_html = self
+            .config
+            .root
+            .join("refs/html")
+            .join(example)
+            .with_extension("html");
+
+        self.update_example(example, &output, &ref_html, "html")
+    }
+
+    fn update_example(&self, example: &str, data: &[u8], path: &Path, kind: &str) -> Result<()> {
+        let ext = path.extension().unwrap().to_string_lossy();
+        let tmp_path = &path.with_extension(format!("tmp.{ext}"));
+        let hash_path = &path.with_extension("hash");
+        let hash = &format!("siphash128_13:{:x}", tinymist_std::hash::hash128(&data));
+
+        let existing_hash = if std::fs::exists(hash_path).context("exists hash ref")? {
+            Some(std::fs::read(hash_path).context("read hash ref")?)
+        } else {
+            None
+        };
+
+        let equal = existing_hash.map(|existing| existing.as_slice() == hash.as_bytes());
+
+        match (self.config.args.update, equal) {
+            // Doesn't exist, create it
+            (_, None) => {}
+            (_, Some(true)) => {
+                eprintln!("   Info example({example}): {kind} matches");
+            }
+            (false, Some(false)) => {
+                eprintln!(" Failed example({example}): {kind} mismatch");
+                eprintln!(
+                    "   Hint example({example}): compare {kind} at {}",
+                    path.display()
+                );
+                self.mark_failed();
+                tinymist_std::fs::paths::write_atomic(tmp_path, data).context("write tmp ref")?;
+                return Ok(());
+            }
+            (true, Some(false)) => {
+                eprintln!("   Info example({example}): updating ref {kind}");
+            }
+        }
+
+        if std::fs::exists(tmp_path).context("exists tmp")? {
+            std::fs::remove_file(tmp_path).context("remove tmp")?;
+        }
+
+        std::fs::create_dir_all(path.parent().context("parent")?).context("create ref")?;
+        tinymist_std::fs::paths::write_atomic(path, data).context("write ref")?;
+        tinymist_std::fs::paths::write_atomic(hash_path, hash).context("write hash ref")?;
+
+        Ok(())
+    }
+
+    fn can_html(&self, doc: Option<&TypstPagedDocument>) -> bool {
+        let Some(doc) = doc else {
+            return false;
+        };
+
+        let label = Label::new(PicoStr::intern("test-html-example"));
+        // todo: error multiple times
+        doc.introspector.query_label(label).is_ok()
+    }
+}
+
+fn with_main(world: &dyn World, id: FileId) -> WorldWithMain<'_> {
+    WorldWithMain { world, main: id }
+}
+
+struct WorldWithMain<'a> {
+    world: &'a dyn World,
+    main: FileId,
+}
+
+impl typst::World for WorldWithMain<'_> {
+    fn main(&self) -> FileId {
+        self.main
+    }
+
+    fn source(&self, id: FileId) -> FileResult<Source> {
+        self.world.source(id)
+    }
+
+    fn library(&self) -> &LazyHash<Library> {
+        self.world.library()
+    }
+
+    fn book(&self) -> &LazyHash<FontBook> {
+        self.world.book()
+    }
+
+    fn file(&self, id: FileId) -> FileResult<Bytes> {
+        self.world.file(id)
+    }
+
+    fn font(&self, index: usize) -> Option<Font> {
+        self.world.font(index)
+    }
+
+    fn today(&self, offset: Option<i64>) -> Option<Datetime> {
+        self.world.today(offset)
+    }
+}
+
+fn get_example_file(world: &dyn World, name: &str, id: FileId, span: Span) -> Result<Source> {
+    let source = world.source(id).context_ut("cannot find file")?;
+    let node = LinkedNode::new(source.root());
+    let leaf = node.find(span).context("cannot find example function")?;
+    let function = node_ancestors(&leaf)
+        .find(|n| n.is::<ast::Closure>())
+        .context("cannot find example function")?;
+    let closure = function.cast::<ast::Closure>().unwrap();
+    if closure.params().children().count() != 0 {
+        bail!("example function must not have parameters");
+    }
+    let included =
+        find_include_expr(name, closure.body()).context("cannot find example function")?;
+    find_source_by_expr(world, id, included).context("cannot find example file")
+}
+
+fn find_include_expr<'a>(name: &str, node: ast::Expr<'a>) -> Option<ast::Expr<'a>> {
+    match node {
+        ast::Expr::Include(inc) => Some(inc.source()),
+        ast::Expr::Code(code) => {
+            let exprs = code.body();
+            if exprs.exprs().count() != 1 {
+                eprintln!("example function must have a single inclusion: {name}");
+                return None;
+            }
+            find_include_expr(name, exprs.exprs().next().unwrap())
+        }
+        _ => {
+            eprintln!("example function must have a single inclusion: {name}");
+            None
+        }
+    }
+}
+
+fn print_diag_or_error<T>(world: &impl SourceWorld, result: Result<T>) -> Result<T> {
+    match result {
+        Ok(v) => Ok(v),
+        Err(err) => {
+            if let Some(diagnostics) = err.diagnostics() {
+                print_diagnostics(world, diagnostics.iter(), DiagnosticFormat::Human)
+                    .context_ut("print diagnostics")?;
+                bail!("");
+            }
+
+            Err(err)
+        }
+    }
+}

docs/tinymist/book.typ

@@ -26,6 +26,7 @@
     - #chapter("guide/completion.typ")[Code Completion]
     - #chapter("feature/export.typ")[Exporting Documents]
     - #chapter("feature/preview.typ")[Document Preview]
+    - #chapter("feature/testing.typ")[Testing]
     - #chapter("feature/language.typ")[Other Features]
     = Service Overview
     #prefix-chapter("overview.typ")[Overview of Service]

docs/tinymist/feature/testing.typ

@@ -0,0 +1,126 @@
+#import "mod.typ": *
+
+#show: book-page.with(title: [Testing Feature])
+
+== IDE Support
+
+You can run tests and check coverage in the IDE or CLI.
+
+== Test Discovery <tinymist-test-discovery>
+
+Given a file, tinymist will try to discover tests related to the file.
+- All dependent files in the same workspace will be checked.
+  - For example, if file `a.typ` contains `import "b.typ"` or `include "b.typ"`, tinymist will check `b.typ` for tests as well.
+- For each file including the entry file itself, tinymist will check the file for tests.
+  - If a file is named `example-*.typ`, it is considered an *example document* and will be compiled using `typst::compile`.
+    - Both pdf export and html export will be called.
+    - If the label `<test-html-example>` can be found in the example file, html export will be called.
+  - Top-level functions will be checked for tests.
+    - If a function is named `test-*`, it is considered a test function and will be called directly.
+    - If a function is named `bench-*`, it is considered a benchmark function and will be called once to collect coverage.
+    - If a function is named `panic-on-*`, it will only pass the test if a panic occurs during execution.
+
+
+Example Entry File:
+```typ
+#import "example-hello-world.typ"
+
+#let test-it() = {
+  "test"
+}
+
+#let panic-on-panic() = {
+  panic("this is a panic")
+}
+```
+
+Example Output:
+
+```
+Found 2 tests and 1 examples
+Running test(test-it)
+Running test(panic-on-panic)
+ Passed test(test-it)
+ Passed test(panic-on-panic)
+Running example(example-hello-world
+ Failed example(example-hello-world): image mismatch
+   Hint example(example-hello-world): compare image at refs/png/example-hello-world.png
+ Passed example(example-hello-world)
+  Info: Written coverage to target/coverage.json ...
+  Info: All test cases passed...
+```
+
+== Visualizing Coverage
+
+- Run and collect file coverage using command `tinymist.profileCurrentFileCoverage` in VS Cod(e,ium).
+- Run and collect test coverage using command `tinymist.profileCurrentTestCoverage` in VS Cod(e,ium).
+  - Check #link(<tinymist-test-discovery>)[Test Suites] to learn how tinymist discovers tests.
+
+VS Cod(e,ium) will show the overall coverage in the editor.
+
+== CLI Support
+
+You can run tests and check coverage in the CLI.
+
+```bash
+tinymist test tests/main.typ
+...
+  Info: All test cases passed...
+```
+
+You can pass same arguments as `typst compile` to `tinymist test`.
+
+== Debugging and behaviors with CLI
+
+If any test fails, the CLI will return a non-zero exit code.
+
+```bash
+tinymist test tests/main.typ
+...
+ Fatal: some test cases failed...
+```
+
+To update the reference files, you can run:
+
+```bash
+tinymist test tests/main.typ --update
+```
+
+To get image files to diff you can use grep to find the image files to update:
+
+```bash
+tinymist test tests/main.typ 2> 2> >(grep Hint) > >(grep "compare image")
+   Hint example(example-hello-world): compare image at target/refs/png/example-hello-world.png
+   Hint example(example-other): compare image at target/refs/png/example-other.png
+```
+
+You can use your favorite image diff tool to compare the images, e.g. `magick compare`.
+
+== Continuous Integration
+
+`tinymist test` only compares hash reference files to check whether content is changed. Therefore, you can ignore the image and html files and only commit the hash files to compare them on CI. Putting the following content in `.gitignore` will help you to ignore the files:
+
+```exclude
+# png files
+refs/png/**/*.png
+# html files
+refs/html/**/*.html
+# hash files
+!refs/**/*.hash
+```
+
+Install `tinymist` on CI and run `tinymist test` to check whether the content is changed.
+
+```yaml
+- name: Install tinymist
+  env:
+    TINYMIST_VERSION: 0.13.x # to test with typst compiler v0.13.x, tinymist v0.14.x for typst v0.14.x, and so on.
+  run: curl --proto '=https' --tlsv1.2 -LsSf https://github.com/Myriad-Dreamin/tinymist/releases/download/${TINYMIST_VERSION}/tinymist-installer.sh | sh
+- name: Run tests (Typst)
+  run: tinymist test tests/main.typ --root . --ppi 144
+- name: Upload artifacts
+  uses: actions/upload-artifact@v4
+  with:
+    name: refs
+    path: refs
+```

editors/vscode/package.json

@@ -1149,6 +1149,11 @@
         "title": "%extension.tinymist.command.tinymist.profileCurrentFileCoverage%",
         "category": "Typst"
       },
+      {
+        "command": "tinymist.profileCurrentTestCoverage",
+        "title": "Profile coverage of the current test module",
+        "category": "Typst"
+      },
       {
         "command": "tinymist.syncLabel",
         "title": "%extension.tinymist.command.tinymist.syncLabel%",

editors/vscode/src/context.ts

@@ -98,6 +98,7 @@ export interface ICommand<T = unknown, R = any> {
 export type IFileLevelCommand = ICommand<FileLevelContext>;
 
 export interface ExecContext extends FileLevelContext {
+  cwd?: string;
   isTTY?: boolean;
   stdout?: (data: Buffer) => void;
   stderr?: (data: Buffer) => void;

editors/vscode/src/features/testing/coverage.ts

@@ -1,43 +1,62 @@
 import * as vscode from "vscode";
-import { IContext } from "../../context";
+import { FileLevelContext, IContext } from "../../context";
 import { VirtualConsole } from "../../util";
 import * as fs from "fs";
 
 export function testingCovActivate(context: IContext, testController: vscode.TestController) {
-  const profileCoverage = testController.createRunProfile(
+  const runTests =
-    "tinymist-profile-coverage",
+    (kind: "cov" | "test") => (request: vscode.TestRunRequest, token: vscode.CancellationToken) =>
+      runCoverageTests(kind, request, token);
+
+  const profileFileCoverage = testController.createRunProfile(
+    "tinymist-profile-file-coverage",
     vscode.TestRunProfileKind.Coverage,
-    runCoverageTests,
+    runTests("cov"),
+  );
+  const profileTestCoverage = testController.createRunProfile(
+    "tinymist-profile-test-coverage",
+    vscode.TestRunProfileKind.Coverage,
+    runTests("test"),
   );
 
-  context.subscriptions.push(testController, profileCoverage);
+  context.subscriptions.push(testController, profileFileCoverage, profileTestCoverage);
+
+  const makeCommand = (kind: "cov" | "test") => async (ctx: FileLevelContext) => {
+    if (!context.isValidEditor(ctx.currentEditor)) {
+      return;
+    }
+    const document = ctx.currentEditor.document;
+
+    const includes = [
+      testController.createTestItem("tinymist-profile", "tinymist-profile", document.uri),
+    ];
+
+    const testRunRequest = new vscode.TestRunRequest(
+      includes,
+      undefined,
+      kind == "cov" ? profileFileCoverage : profileTestCoverage,
+      false,
+      true,
+    );
+
+    const cc = new vscode.CancellationTokenSource();
+    runCoverageTests(kind, testRunRequest, cc.token);
+  };
 
   context.registerFileLevelCommand({
     command: "tinymist.profileCurrentFileCoverage",
-    execute: async (ctx) => {
+    execute: makeCommand("cov"),
-      if (!context.isValidEditor(ctx.currentEditor)) {
+  });
-        return;
+  context.registerFileLevelCommand({
-      }
+    command: "tinymist.profileCurrentTestCoverage",
-      const document = ctx.currentEditor.document;
+    execute: makeCommand("test"),
-
-      const includes = [
-        testController.createTestItem("tinymist-profile", "tinymist-profile", document.uri),
-      ];
-
-      const testRunRequest = new vscode.TestRunRequest(
-        includes,
-        undefined,
-        profileCoverage,
-        false,
-        true,
-      );
-
-      const cc = new vscode.CancellationTokenSource();
-      runCoverageTests(testRunRequest, cc.token);
-    },
   });
 
-  async function runCoverageTests(request: vscode.TestRunRequest, token: vscode.CancellationToken) {
+  async function runCoverageTests(
+    testKind: "cov" | "test",
+    request: vscode.TestRunRequest,
+    token: vscode.CancellationToken,
+  ) {
     const testRun = testController.createTestRun(request);
     if (request.include?.length !== 1) {
       context.showErrorMessage("Invalid tinymist test run request");
@@ -51,6 +70,12 @@ export function testingCovActivate(context: IContext, testController: vscode.Tes
       return;
     }
     testRun.started(item);
+    const cwd = context.getRootForUri(uri);
+    if (!cwd) {
+      context.showErrorMessage(`tinymist cannot find root for uri: ${uri}`);
+      return;
+    }
+    const rootArgs = ["--root", cwd];
 
     const failed = (msg: string) => {
       testRun.failed(item, new vscode.TestMessage(msg));
@@ -83,6 +108,7 @@ export function testingCovActivate(context: IContext, testController: vscode.Tes
 
     const coverageTask = executable.execute(
       {
+        cwd,
         killer,
         isTTY: true,
         stdout: (data: Buffer) => {
@@ -92,7 +118,7 @@ export function testingCovActivate(context: IContext, testController: vscode.Tes
           vc.write(data.toString("utf8"));
         },
       },
-      ["cov", uri.fsPath],
+      [testKind, ...rootArgs, uri.fsPath],
     );
 
     const detailsFut = coverageTask.then<Map<string, vscode.FileCoverageDetail[]>>((res) => {
@@ -101,12 +127,12 @@ export function testingCovActivate(context: IContext, testController: vscode.Tes
         return details;
       }
 
-      const cov_path = "target/coverage.json";
+      const covPath = vscode.Uri.joinPath(vscode.Uri.file(cwd!), "target/coverage.json").fsPath;
-      if (!fs.existsSync(cov_path)) {
+      if (!fs.existsSync(covPath)) {
         return details;
       }
 
-      const cov = fs.readFileSync(cov_path, "utf8");
+      const cov = fs.readFileSync(covPath, "utf8");
       const cov_json: Record<string, vscode.StatementCoverage[]> = JSON.parse(cov);
       for (const [k, v] of Object.entries(cov_json)) {
         details.set(