feat(test): create coverage reports when --coverage specified in deno test (#28260)

This PR updates the behavior of `deno test --coverage` option. Now if
`--coverage` option is specified, `deno test` command automatically
shows summary report in the terminal, and generates the lcov report in
`$coverage_dir/lcov.info` and html report in `$coverage_dir/html/`

This change also adds `--coverage-raw-data-only` flag, which prevents
the above reports generated, instead only generates the raw json
coverage data (which is the same as current behavior)

cli/args/flags.rs

@@ -409,6 +409,7 @@ pub struct TestFlags {
   pub doc: bool,
   pub no_run: bool,
   pub coverage_dir: Option<String>,
+  pub coverage_raw_data_only: bool,
   pub clean: bool,
   pub fail_fast: Option<NonZeroUsize>,
   pub files: FileFlags,
@@ -3241,6 +3242,13 @@ or <c>**/__tests__/**</>:
   <p(245)>This option can also be set via the DENO_COVERAGE_DIR environment variable."))
           .help_heading(TEST_HEADING),
       )
+      .arg(
+        Arg::new("coverage-raw-data-only")
+          .long("coverage-raw-data-only")
+          .help("Only collect raw coverage data, without generating a report")
+          .action(ArgAction::SetTrue)
+          .help_heading(TEST_HEADING),
+      )
       .arg(
         Arg::new("clean")
           .long("clean")
@@ -5536,6 +5544,7 @@ fn test_parse(
     no_run,
     doc,
     coverage_dir: matches.remove_one::<String>("coverage"),
+    coverage_raw_data_only: matches.get_flag("coverage-raw-data-only"),
     clean,
     fail_fast,
     files: FileFlags { include, ignore },
@@ -9478,6 +9487,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: true,
           coverage_dir: Some("cov".to_string()),
+          coverage_raw_data_only: false,
           clean: true,
           watch: Default::default(),
           reporter: Default::default(),
@@ -9562,6 +9572,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Default::default(),
           reporter: Default::default(),
@@ -9605,6 +9616,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Default::default(),
           reporter: Default::default(),
@@ -9742,6 +9754,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Default::default(),
           reporter: Default::default(),
@@ -9778,6 +9791,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Some(Default::default()),
           reporter: Default::default(),
@@ -9813,6 +9827,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Some(Default::default()),
           reporter: Default::default(),
@@ -9850,6 +9865,7 @@ mod tests {
           concurrent_jobs: None,
           trace_leaks: false,
           coverage_dir: None,
+          coverage_raw_data_only: false,
           clean: false,
           watch: Some(WatchFlagsWithPaths {
             hmr: false,

cli/main.rs

@@ -144,8 +144,17 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
         tools::compile::compile(flags, compile_flags).await
       }
     }),
-    DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async {
+    DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async move {
-      tools::coverage::cover_files(flags, coverage_flags)
+      let reporter = crate::tools::coverage::reporter::create(coverage_flags.r#type.clone());
+      tools::coverage::cover_files(
+        flags,
+        coverage_flags.files.include,
+        coverage_flags.files.ignore,
+        coverage_flags.include,
+        coverage_flags.exclude,
+        coverage_flags.output,
+        &[&*reporter]
+      )
     }),
     DenoSubcommand::Fmt(fmt_flags) => {
       spawn_subcommand(
@@ -285,7 +294,8 @@ async fn run_subcommand(flags: Arc<Flags>) -> Result<i32, AnyError> {
     DenoSubcommand::Test(test_flags) => {
       spawn_subcommand(async {
         if let Some(ref coverage_dir) = test_flags.coverage_dir {
-          if test_flags.clean {
+          if !test_flags.coverage_raw_data_only || test_flags.clean {
+            // Keeps coverage_dir contents only when --coverage-raw-data-only is set and --clean is not set
             let _ = std::fs::remove_dir_all(coverage_dir);
           }
           std::fs::create_dir_all(coverage_dir)

cli/tools/coverage/mod.rs

@@ -27,6 +27,7 @@ use deno_error::JsErrorBox;
 use deno_resolver::npm::DenoInNpmPackageChecker;
 use node_resolver::InNpmPackageChecker;
 use regex::Regex;
+use reporter::CoverageReporter;
 use text_lines::TextLines;
 use uuid::Uuid;
 
@@ -35,7 +36,6 @@ use self::ignore_directives::lex_comments;
 use self::ignore_directives::parse_next_ignore_directives;
 use self::ignore_directives::parse_range_ignore_directives;
 use crate::args::CliOptions;
-use crate::args::CoverageFlags;
 use crate::args::FileFlags;
 use crate::args::Flags;
 use crate::cdp;
@@ -49,7 +49,7 @@ use crate::util::text_encoding::source_map_from_code;
 mod ignore_directives;
 mod merge;
 mod range_tree;
-mod reporter;
+pub mod reporter;
 mod util;
 use merge::ProcessCoverage;
 
@@ -596,9 +596,14 @@ fn filter_coverages(
 
 pub fn cover_files(
   flags: Arc<Flags>,
-  coverage_flags: CoverageFlags,
+  files_include: Vec<String>,
+  files_ignore: Vec<String>,
+  include: Vec<String>,
+  exclude: Vec<String>,
+  output: Option<String>,
+  reporters: &[&dyn CoverageReporter],
 ) -> Result<(), AnyError> {
-  if coverage_flags.files.include.is_empty() {
+  if files_include.is_empty() {
     return Err(anyhow!("No matching coverage profiles found"));
   }
 
@@ -609,26 +614,21 @@ pub fn cover_files(
   let emitter = factory.emitter()?;
   let cjs_tracker = factory.cjs_tracker()?;
 
-  assert!(!coverage_flags.files.include.is_empty());
-
   // Use the first include path as the default output path.
-  let coverage_root = cli_options
+  let coverage_root = cli_options.initial_cwd().join(&files_include[0]);
-    .initial_cwd()
-    .join(&coverage_flags.files.include[0]);
   let script_coverages = collect_coverages(
     cli_options,
-    coverage_flags.files,
+    FileFlags {
+      include: files_include,
+      ignore: files_ignore,
+    },
     cli_options.initial_cwd(),
   )?;
   if script_coverages.is_empty() {
     return Err(anyhow!("No coverage files found"));
   }
-  let script_coverages = filter_coverages(
+  let script_coverages =
-    script_coverages,
+    filter_coverages(script_coverages, include, exclude, in_npm_pkg_checker);
-    coverage_flags.include,
-    coverage_flags.exclude,
-    in_npm_pkg_checker,
-  );
   if script_coverages.is_empty() {
     return Err(anyhow!("No covered files included in the report"));
   }
@@ -645,9 +645,7 @@ pub fn cover_files(
     vec![]
   };
 
-  let mut reporter = reporter::create(coverage_flags.r#type);
+  let out_mode = match output {
-
-  let out_mode = match coverage_flags.output {
     Some(ref path) => match File::create(path) {
       Ok(_) => Some(PathBuf::from(path)),
       Err(e) => {
@@ -742,7 +740,9 @@ pub fn cover_files(
     }
   }
 
+  for reporter in reporters {
     reporter.done(&coverage_root, &file_reports);
+  }
 
   Ok(())
 }

cli/tools/coverage/reporter.rs

@@ -42,7 +42,7 @@ pub fn create(kind: CoverageType) -> Box<dyn CoverageReporter + Send> {
 
 pub trait CoverageReporter {
   fn done(
-    &mut self,
+    &self,
     coverage_root: &Path,
     file_reports: &[(CoverageReport, String)],
   );
@@ -105,7 +105,7 @@ pub trait CoverageReporter {
   }
 }
 
-struct SummaryCoverageReporter {}
+pub struct SummaryCoverageReporter {}
 
 #[allow(clippy::print_stdout)]
 impl SummaryCoverageReporter {
@@ -172,7 +172,7 @@ impl SummaryCoverageReporter {
 #[allow(clippy::print_stdout)]
 impl CoverageReporter for SummaryCoverageReporter {
   fn done(
-    &mut self,
+    &self,
     _coverage_root: &Path,
     file_reports: &[(CoverageReport, String)],
   ) {
@@ -206,17 +206,30 @@ impl CoverageReporter for SummaryCoverageReporter {
   }
 }
 
-struct LcovCoverageReporter {}
+pub struct LcovCoverageReporter {}
 
 impl CoverageReporter for LcovCoverageReporter {
   fn done(
-    &mut self,
+    &self,
     _coverage_root: &Path,
     file_reports: &[(CoverageReport, String)],
   ) {
     file_reports.iter().for_each(|(report, file_text)| {
       self.report(report, file_text).unwrap();
     });
+    if let Some((report, _)) = file_reports.first() {
+      if let Some(ref output) = report.output {
+        if let Ok(path) = output.canonicalize() {
+          let url = Url::from_file_path(path).unwrap();
+          log::info!("Lcov coverage report has been generated at {}", url);
+        } else {
+          log::error!(
+            "Failed to resolve the output path of Lcov report: {}",
+            output.display()
+          );
+        }
+      }
+    }
   }
 }
 
@@ -226,7 +239,7 @@ impl LcovCoverageReporter {
   }
 
   fn report(
-    &mut self,
+    &self,
     coverage_report: &CoverageReport,
     _file_text: &str,
   ) -> Result<(), AnyError> {
@@ -320,7 +333,7 @@ struct DetailedCoverageReporter {}
 
 impl CoverageReporter for DetailedCoverageReporter {
   fn done(
-    &mut self,
+    &self,
     _coverage_root: &Path,
     file_reports: &[(CoverageReport, String)],
   ) {
@@ -337,7 +350,7 @@ impl DetailedCoverageReporter {
   }
 
   fn report(
-    &mut self,
+    &self,
     coverage_report: &CoverageReport,
     file_text: &str,
   ) -> Result<(), AnyError> {
@@ -398,11 +411,11 @@ impl DetailedCoverageReporter {
   }
 }
 
-struct HtmlCoverageReporter {}
+pub struct HtmlCoverageReporter {}
 
 impl CoverageReporter for HtmlCoverageReporter {
   fn done(
-    &mut self,
+    &self,
     coverage_root: &Path,
     file_reports: &[(CoverageReport, String)],
   ) {

cli/tools/test/mod.rs

@@ -12,6 +12,7 @@ use std::future::poll_fn;
 use std::io::Write;
 use std::num::NonZeroUsize;
 use std::path::Path;
+use std::path::PathBuf;
 use std::sync::atomic::AtomicBool;
 use std::sync::atomic::AtomicUsize;
 use std::sync::atomic::Ordering;
@@ -109,6 +110,8 @@ use reporters::PrettyTestReporter;
 use reporters::TapTestReporter;
 use reporters::TestReporter;
 
+use crate::tools::coverage::cover_files;
+use crate::tools::coverage::reporter;
 use crate::tools::test::channel::ChannelClosedError;
 
 /// How many times we're allowed to spin the event loop before considering something a leak.
@@ -1562,7 +1565,7 @@ pub async fn run_tests(
   flags: Arc<Flags>,
   test_flags: TestFlags,
 ) -> Result<(), AnyError> {
-  let factory = CliFactory::from_flags(flags);
+  let factory = CliFactory::from_flags(flags.clone());
   let cli_options = factory.cli_options()?;
   let workspace_test_options =
     cli_options.resolve_workspace_test_options(&test_flags);
@@ -1650,6 +1653,34 @@ pub async fn run_tests(
   )
   .await?;
 
+  if test_flags.coverage_raw_data_only {
+    return Ok(());
+  }
+
+  if let Some(ref coverage) = test_flags.coverage_dir {
+    let reporters: [&dyn reporter::CoverageReporter; 3] = [
+      &reporter::SummaryCoverageReporter::new(),
+      &reporter::LcovCoverageReporter::new(),
+      &reporter::HtmlCoverageReporter::new(),
+    ];
+    if let Err(err) = cover_files(
+      flags,
+      vec![coverage.clone()],
+      vec![],
+      vec![],
+      vec![],
+      Some(
+        PathBuf::from(coverage)
+          .join("lcov.info")
+          .to_string_lossy()
+          .to_string(),
+      ),
+      &reporters,
+    ) {
+      log::info!("Error generating coverage report: {}", err);
+    }
+  }
+
   Ok(())
 }
 

tests/integration/coverage_tests.rs

@@ -665,7 +665,7 @@ fn test_collect_summary_with_no_matches() {
   let temp_dir: &TempDir = context.temp_dir();
   let temp_dir_path: PathRef = PathRef::new(temp_dir.path().join("cov"));
 
-  let empty_test_dir: PathRef = temp_dir_path.join("empty_dir");
+  let empty_test_dir: PathRef = temp_dir.path().join("empty_dir");
   empty_test_dir.create_dir_all();
 
   let output: util::TestCommandOutput = context

tests/specs/coverage/coverage_raw_data_only/__test__.jsonc

@@ -0,0 +1,10 @@
+{
+  "tempDir": true,
+  "steps": [
+    {
+      "args": "test --doc --coverage --coverage-raw-data-only",
+      "output": "main.out",
+      "exitCode": 0
+    }
+  ]
+}

tests/specs/coverage/coverage_raw_data_only/main.out

@@ -0,0 +1,6 @@
+Check [WILDCARD]/test.ts
+running 1 test from ./test.ts
+add() ... ok ([WILDCARD])
+
+ok | 1 passed | 0 failed ([WILDCARD])
+

tests/specs/coverage/coverage_raw_data_only/source.ts

@@ -0,0 +1,3 @@
+export function add(a: number, b: number): number {
+  return a + b;
+}

tests/specs/coverage/coverage_raw_data_only/test.ts

@@ -0,0 +1,7 @@
+import { add } from "./source.ts";
+
+Deno.test("add()", () => {
+  if (add(1, 2) !== 3) {
+    throw new Error("test failed");
+  }
+});

tests/specs/coverage/default_reports/__test__.jsonc

@@ -0,0 +1,20 @@
+{
+  "tempDir": true,
+  "steps": [
+    {
+      "args": "test --coverage --doc",
+      "output": "main.out",
+      "exitCode": 0
+    },
+    {
+      "args": "run -A cat.ts coverage/lcov.info",
+      "output": "lcov_info.out",
+      "exitCode": 0
+    },
+    {
+      "args": "run -A cat.ts coverage/html/index.html",
+      "output": "html_index_html.out",
+      "exitCode": 0
+    }
+  ]
+}

tests/specs/coverage/default_reports/cat.ts

@@ -0,0 +1 @@
+console.log(await Deno.readTextFile(Deno.args[0]));

tests/specs/coverage/default_reports/html_index_html.out

@@ -0,0 +1,2 @@
+<!doctype html>
+      <html>[WILDCARD]</html>

tests/specs/coverage/default_reports/lcov_info.out

@@ -0,0 +1,4 @@
+SF:[WILDCARD]source.ts
+[WILDCARD]
+end_of_record
+

tests/specs/coverage/default_reports/main.out

@@ -0,0 +1,18 @@
+Check [WILDCARD]/test.ts
+Check [WILDCARD]/source.ts$[WILDCARD].ts
+running 1 test from ./test.ts
+add() ... ok ([WILDCARD])
+running 1 test from ./source.ts$[WILDCARD].ts
+file:///[WILDCARD]/source.ts$[WILDCARD].ts ... ok ([WILDCARD])
+
+ok | 2 passed | 0 failed ([WILDCARD])
+
+--------------------------------
+File       | Branch % | Line % |
+--------------------------------
+ source.ts |    100.0 |  100.0 |
+--------------------------------
+ All files |    100.0 |  100.0 |
+--------------------------------
+Lcov coverage report has been generated at file://[WILDCARD]/coverage/lcov.info
+HTML coverage report has been generated at file://[WILDCARD]/coverage/html/index.html

tests/specs/coverage/default_reports/source.ts

@@ -0,0 +1,9 @@
+/**
+ * @example Usage
+ * ```ts
+ * add(1, 2); // 3
+ * ```
+ */
+export function add(a: number, b: number): number {
+  return a + b;
+}

tests/specs/coverage/default_reports/test.ts

@@ -0,0 +1,7 @@
+import { add } from "./source.ts";
+
+Deno.test("add()", () => {
+  if (add(1, 2) !== 3) {
+    throw new Error("test failed");
+  }
+});

tests/specs/coverage/filter_doc_testing_urls/test_coverage.out

@@ -7,3 +7,12 @@ file:///[WILDCARD]/source.ts$[WILDCARD].ts ... ok ([WILDCARD])
 
 ok | 2 passed | 0 failed ([WILDCARD])
 
+--------------------------------
+File       | Branch % | Line % |
+--------------------------------
+ source.ts |    100.0 |  100.0 |
+--------------------------------
+ All files |    100.0 |  100.0 |
+--------------------------------
+Lcov coverage report has been generated at file://[WILDCARD]/coverage/lcov.info
+HTML coverage report has been generated at file://[WILDCARD]/coverage/html/index.html

tests/specs/coverage/no_files_after_filter/test_coverage.out

@@ -4,3 +4,4 @@ test ... ok ([WILDCARD])
 
 ok | 1 passed | 0 failed ([WILDCARD])
 
+Error generating coverage report: No covered files included in the report

tests/specs/test/clean_flag/main.js

@@ -2,7 +2,7 @@ import { emptyDir } from "@std/fs/empty-dir";
 
 const DIR = "./coverage";
 const COMMAND = new Deno.Command(Deno.execPath(), {
-  args: ["test", "--coverage", "--clean"],
+  args: ["test", "--coverage", "--clean", "--coverage-raw-data-only"],
   stdout: "null",
 });