feat(unstable): support caching npm dependencies only as they're needed (#27300)

Currently deno eagerly caches all npm packages in the workspace's npm
resolution. So, for instance, running a file `foo.ts` that imports
`npm:chalk` will also install all dependencies listed in `package.json`
and all `npm` dependencies listed in the lockfile.

This PR refactors things to give more control over when and what npm
packages are automatically cached while building the module graph.

After this PR, by default the current behavior is unchanged _except_ for
`deno install --entrypoint`, which will only cache npm packages used by
the given entrypoint. For the other subcommands, this behavior can be
enabled with `--unstable-npm-lazy-caching`


Fixes #25782.

---------

Signed-off-by: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com>
Co-authored-by: Luca Casonato <hello@lcas.dev>

cli/graph_util.rs

@@ -4,6 +4,7 @@ use crate::args::config_to_deno_graph_workspace_member;
 use crate::args::jsr_url;
 use crate::args::CliLockfile;
 use crate::args::CliOptions;
+pub use crate::args::NpmCachingStrategy;
 use crate::args::DENO_DISABLE_PEDANTIC_NODE_WARNINGS;
 use crate::cache;
 use crate::cache::FetchCacher;
@@ -218,6 +219,7 @@ pub struct CreateGraphOptions<'a> {
   pub is_dynamic: bool,
   /// Specify `None` to use the default CLI loader.
   pub loader: Option<&'a mut dyn Loader>,
+  pub npm_caching: NpmCachingStrategy,
 }
 
 pub struct ModuleGraphCreator {
@@ -246,10 +248,11 @@ impl ModuleGraphCreator {
     &self,
     graph_kind: GraphKind,
     roots: Vec<ModuleSpecifier>,
+    npm_caching: NpmCachingStrategy,
   ) -> Result<deno_graph::ModuleGraph, AnyError> {
     let mut cache = self.module_graph_builder.create_graph_loader();
     self
-      .create_graph_with_loader(graph_kind, roots, &mut cache)
+      .create_graph_with_loader(graph_kind, roots, &mut cache, npm_caching)
       .await
   }
 
@@ -258,6 +261,7 @@ impl ModuleGraphCreator {
     graph_kind: GraphKind,
     roots: Vec<ModuleSpecifier>,
     loader: &mut dyn Loader,
+    npm_caching: NpmCachingStrategy,
   ) -> Result<ModuleGraph, AnyError> {
     self
       .create_graph_with_options(CreateGraphOptions {
@@ -265,6 +269,7 @@ impl ModuleGraphCreator {
         graph_kind,
         roots,
         loader: Some(loader),
+        npm_caching,
       })
       .await
   }
@@ -317,6 +322,7 @@ impl ModuleGraphCreator {
         graph_kind: deno_graph::GraphKind::All,
         roots,
         loader: Some(&mut publish_loader),
+        npm_caching: self.options.default_npm_caching_strategy(),
       })
       .await?;
     self.graph_valid(&graph)?;
@@ -376,6 +382,7 @@ impl ModuleGraphCreator {
         graph_kind,
         roots,
         loader: None,
+        npm_caching: self.options.default_npm_caching_strategy(),
       })
       .await?;
 
@@ -565,7 +572,8 @@ impl ModuleGraphBuilder {
     };
     let cli_resolver = &self.resolver;
     let graph_resolver = self.create_graph_resolver()?;
-    let graph_npm_resolver = cli_resolver.create_graph_npm_resolver();
+    let graph_npm_resolver =
+      cli_resolver.create_graph_npm_resolver(options.npm_caching);
     let maybe_file_watcher_reporter = self
       .maybe_file_watcher_reporter
       .as_ref()
@@ -592,6 +600,7 @@ impl ModuleGraphBuilder {
           resolver: Some(&graph_resolver),
           locker: locker.as_mut().map(|l| l as _),
         },
+        options.npm_caching,
       )
       .await
   }
@@ -602,6 +611,7 @@ impl ModuleGraphBuilder {
     roots: Vec<ModuleSpecifier>,
     loader: &'a mut dyn deno_graph::source::Loader,
     options: deno_graph::BuildOptions<'a>,
+    npm_caching: NpmCachingStrategy,
   ) -> Result<(), AnyError> {
     // ensure an "npm install" is done if the user has explicitly
     // opted into using a node_modules directory
@@ -612,7 +622,13 @@ impl ModuleGraphBuilder {
       .unwrap_or(false)
     {
       if let Some(npm_resolver) = self.npm_resolver.as_managed() {
-        npm_resolver.ensure_top_level_package_json_install().await?;
+        let already_done =
+          npm_resolver.ensure_top_level_package_json_install().await?;
+        if !already_done && matches!(npm_caching, NpmCachingStrategy::Eager) {
+          npm_resolver
+            .cache_packages(crate::npm::PackageCaching::All)
+            .await?;
+        }
       }
     }
 
@@ -701,7 +717,9 @@ impl ModuleGraphBuilder {
     let parser = self.parsed_source_cache.as_capturing_parser();
     let cli_resolver = &self.resolver;
     let graph_resolver = self.create_graph_resolver()?;
-    let graph_npm_resolver = cli_resolver.create_graph_npm_resolver();
+    let graph_npm_resolver = cli_resolver.create_graph_npm_resolver(
+      self.cli_options.default_npm_caching_strategy(),
+    );
 
     graph.build_fast_check_type_graph(
       deno_graph::BuildFastCheckTypeGraphOptions {