feat(node): Support executing npm package lifecycle scripts (preinstall/install/postinstall) (#24487)

Adds support for running npm package lifecycle scripts, opted into via a
new `--allow-scripts` flag.

With this PR, when running `deno cache` (or `DENO_FUTURE=1 deno
install`) you can specify the `--allow-scripts=pkg1,pkg2` flag to run
lifecycle scripts attached to the given packages.

Note at the moment this only works when `nodeModulesDir` is true (using
the local resolver).

When a package with un-run lifecycle scripts is encountered, we emit a
warning suggesting things may not work and to try running lifecycle
scripts. Additionally, if a package script implicitly requires
`node-gyp` and it's not found on the system, we emit a warning.

Extra things in this PR:
- Extracted out bits of `task.rs` into a separate module for reuse
- Added a couple fields to `process.config` in order to support
`node-gyp` (it relies on a few variables being there)
- Drive by fix to downloading new npm packages to test registry

---

TODO:
- [x] validation for allow-scripts args (make sure it looks like an npm
package)
- [x] make allow-scripts matching smarter
- [ ] figure out what issues this closes

---
Review notes:
- This adds a bunch of deps to our test registry due to using
`node-gyp`, so it's pretty noisy

cli/npm/managed/resolvers/local.rs

@@ -16,8 +16,11 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::sync::Arc;
 
+use crate::args::LifecycleScriptsConfig;
+use crate::args::PackagesAllowedScripts;
 use async_trait::async_trait;
 use deno_ast::ModuleSpecifier;
+use deno_core::anyhow;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::futures::stream::FuturesUnordered;
@@ -68,6 +71,7 @@ pub struct LocalNpmPackageResolver {
   root_node_modules_url: Url,
   system_info: NpmSystemInfo,
   registry_read_permission_checker: RegistryReadPermissionChecker,
+  lifecycle_scripts: LifecycleScriptsConfig,
 }
 
 impl LocalNpmPackageResolver {
@@ -81,6 +85,7 @@ impl LocalNpmPackageResolver {
     tarball_cache: Arc<TarballCache>,
     node_modules_folder: PathBuf,
     system_info: NpmSystemInfo,
+    lifecycle_scripts: LifecycleScriptsConfig,
   ) -> Self {
     Self {
       cache,
@@ -97,6 +102,7 @@ impl LocalNpmPackageResolver {
         .unwrap(),
       root_node_modules_path: node_modules_folder,
       system_info,
+      lifecycle_scripts,
     }
   }
 
@@ -245,6 +251,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
       &self.tarball_cache,
       &self.root_node_modules_path,
       &self.system_info,
+      &self.lifecycle_scripts,
     )
     .await
   }
@@ -260,7 +267,131 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
   }
 }
 
+// take in all (non copy) packages from snapshot,
+// and resolve the set of available binaries to create
+// custom commands available to the task runner
+fn resolve_baseline_custom_commands(
+  snapshot: &NpmResolutionSnapshot,
+  packages: &[NpmResolutionPackage],
+  local_registry_dir: &Path,
+) -> Result<crate::task_runner::TaskCustomCommands, AnyError> {
+  let mut custom_commands = crate::task_runner::TaskCustomCommands::new();
+  custom_commands
+    .insert("npx".to_string(), Rc::new(crate::task_runner::NpxCommand));
+
+  custom_commands
+    .insert("npm".to_string(), Rc::new(crate::task_runner::NpmCommand));
+
+  custom_commands
+    .insert("node".to_string(), Rc::new(crate::task_runner::NodeCommand));
+
+  custom_commands.insert(
+    "node-gyp".to_string(),
+    Rc::new(crate::task_runner::NodeGypCommand),
+  );
+
+  // TODO: this recreates the bin entries which could be redoing some work, but the ones
+  // we compute earlier in `sync_resolution_with_fs` may not be exhaustive (because we skip
+  // doing it for packages that are set up already.
+  // realistically, scripts won't be run very often so it probably isn't too big of an issue.
+  resolve_custom_commands_from_packages(
+    custom_commands,
+    snapshot,
+    packages,
+    local_registry_dir,
+  )
+}
+
+// resolves the custom commands from an iterator of packages
+// and adds them to the existing custom commands.
+// note that this will overwrite any existing custom commands
+fn resolve_custom_commands_from_packages<
+  'a,
+  P: IntoIterator<Item = &'a NpmResolutionPackage>,
+>(
+  mut commands: crate::task_runner::TaskCustomCommands,
+  snapshot: &'a NpmResolutionSnapshot,
+  packages: P,
+  local_registry_dir: &Path,
+) -> Result<crate::task_runner::TaskCustomCommands, AnyError> {
+  let mut bin_entries = bin_entries::BinEntries::new();
+  for package in packages {
+    let package_path =
+      local_node_modules_package_path(local_registry_dir, package);
+
+    if package.bin.is_some() {
+      bin_entries.add(package.clone(), package_path);
+    }
+  }
+  let bins = bin_entries.into_bin_files(snapshot);
+  for (bin_name, script_path) in bins {
+    commands.insert(
+      bin_name.clone(),
+      Rc::new(crate::task_runner::NodeModulesFileRunCommand {
+        command_name: bin_name,
+        path: script_path,
+      }),
+    );
+  }
+
+  Ok(commands)
+}
+
+fn local_node_modules_package_path(
+  local_registry_dir: &Path,
+  package: &NpmResolutionPackage,
+) -> PathBuf {
+  local_registry_dir
+    .join(get_package_folder_id_folder_name(
+      &package.get_package_cache_folder_id(),
+    ))
+    .join("node_modules")
+    .join(&package.id.nv.name)
+}
+
+// resolves the custom commands from the dependencies of a package
+// and adds them to the existing custom commands.
+// note that this will overwrite any existing custom commands.
+fn resolve_custom_commands_from_deps(
+  baseline: crate::task_runner::TaskCustomCommands,
+  package: &NpmResolutionPackage,
+  snapshot: &NpmResolutionSnapshot,
+  local_registry_dir: &Path,
+) -> Result<crate::task_runner::TaskCustomCommands, AnyError> {
+  resolve_custom_commands_from_packages(
+    baseline,
+    snapshot,
+    package
+      .dependencies
+      .values()
+      .map(|id| snapshot.package_from_id(id).unwrap()),
+    local_registry_dir,
+  )
+}
+
+fn can_run_scripts(
+  allow_scripts: &PackagesAllowedScripts,
+  package_nv: &PackageNv,
+) -> bool {
+  match allow_scripts {
+    PackagesAllowedScripts::All => true,
+    // TODO: make this more correct
+    PackagesAllowedScripts::Some(allow_list) => allow_list.iter().any(|s| {
+      let s = s.strip_prefix("npm:").unwrap_or(s);
+      s == package_nv.name || s == package_nv.to_string()
+    }),
+    PackagesAllowedScripts::None => false,
+  }
+}
+
+fn has_lifecycle_scripts(package: &NpmResolutionPackage) -> bool {
+  package.scripts.contains_key("preinstall")
+    || package.scripts.contains_key("install")
+    || package.scripts.contains_key("postinstall")
+}
+
 /// Creates a pnpm style folder structure.
+#[allow(clippy::too_many_arguments)]
 async fn sync_resolution_with_fs(
   snapshot: &NpmResolutionSnapshot,
   cache: &Arc<NpmCache>,
@@ -269,6 +400,7 @@ async fn sync_resolution_with_fs(
   tarball_cache: &Arc<TarballCache>,
   root_node_modules_dir_path: &Path,
   system_info: &NpmSystemInfo,
+  lifecycle_scripts: &LifecycleScriptsConfig,
 ) -> Result<(), AnyError> {
   if snapshot.is_empty() && pkg_json_deps_provider.workspace_pkgs().is_empty() {
     return Ok(()); // don't create the directory
@@ -307,6 +439,8 @@ async fn sync_resolution_with_fs(
   let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> =
     HashMap::with_capacity(package_partitions.packages.len());
   let bin_entries = Rc::new(RefCell::new(bin_entries::BinEntries::new()));
+  let mut packages_with_scripts = Vec::with_capacity(2);
+  let mut packages_with_scripts_not_run = Vec::new();
   for package in &package_partitions.packages {
     if let Some(current_pkg) =
       newest_packages_by_name.get_mut(&package.id.nv.name)
@@ -331,6 +465,7 @@ async fn sync_resolution_with_fs(
       // are forced to be recreated
       setup_cache.remove_dep(&package_folder_name);
 
+      let folder_path = folder_path.clone();
       let bin_entries_to_setup = bin_entries.clone();
       cache_futures.push(async move {
         tarball_cache
@@ -368,6 +503,24 @@ async fn sync_resolution_with_fs(
         Ok::<_, AnyError>(())
       });
     }
+
+    if has_lifecycle_scripts(package) {
+      let scripts_run = folder_path.join(".scripts-run");
+      if can_run_scripts(&lifecycle_scripts.allowed, &package.id.nv) {
+        if !scripts_run.exists() {
+          let sub_node_modules = folder_path.join("node_modules");
+          let package_path =
+            join_package_name(&sub_node_modules, &package.id.nv.name);
+          packages_with_scripts.push((
+            package.clone(),
+            package_path,
+            scripts_run,
+          ));
+        }
+      } else if !scripts_run.exists() {
+        packages_with_scripts_not_run.push(package.id.nv.clone());
+      }
+    }
   }
 
   while let Some(result) = cache_futures.next().await {
@@ -509,6 +662,73 @@ async fn sync_resolution_with_fs(
     }
   }
 
+  if !packages_with_scripts.is_empty() {
+    // get custom commands for each bin available in the node_modules dir (essentially
+    // the scripts that are in `node_modules/.bin`)
+    let base = resolve_baseline_custom_commands(
+      snapshot,
+      &package_partitions.packages,
+      &deno_local_registry_dir,
+    )?;
+    let init_cwd = lifecycle_scripts.initial_cwd.as_deref().unwrap();
+
+    for (package, package_path, scripts_run_path) in packages_with_scripts {
+      // add custom commands for binaries from the package's dependencies. this will take precedence over the
+      // baseline commands, so if the package relies on a bin that conflicts with one higher in the dependency tree, the
+      // correct bin will be used.
+      let custom_commands = resolve_custom_commands_from_deps(
+        base.clone(),
+        &package,
+        snapshot,
+        &deno_local_registry_dir,
+      )?;
+      for script_name in ["preinstall", "install", "postinstall"] {
+        if let Some(script) = package.scripts.get(script_name) {
+          let exit_code =
+            crate::task_runner::run_task(crate::task_runner::RunTaskOptions {
+              task_name: script_name,
+              script,
+              cwd: &package_path,
+              env_vars: crate::task_runner::real_env_vars(),
+              custom_commands: custom_commands.clone(),
+              init_cwd,
+              argv: &[],
+              root_node_modules_dir: Some(root_node_modules_dir_path),
+            })
+            .await?;
+          if exit_code != 0 {
+            anyhow::bail!(
+              "script '{}' in '{}' failed with exit code {}",
+              script_name,
+              package.id.nv,
+              exit_code,
+            );
+          }
+        }
+      }
+      fs::write(scripts_run_path, "")?;
+    }
+  }
+
+  if !packages_with_scripts_not_run.is_empty() {
+    let (maybe_install, maybe_install_example) = if *crate::args::DENO_FUTURE {
+      (
+        " or `deno install`",
+        " or `deno install --allow-scripts=pkg1,pkg2`",
+      )
+    } else {
+      ("", "")
+    };
+    let packages = packages_with_scripts_not_run
+      .iter()
+      .map(|p| p.to_string())
+      .collect::<Vec<_>>()
+      .join(", ");
+    log::warn!("{}: Packages contained npm lifecycle scripts (preinstall/install/postinstall) that were not executed.
+    This may cause the packages to not work correctly. To run them, use the `--allow-scripts` flag with `deno cache`{maybe_install}
+    (e.g. `deno cache --allow-scripts=pkg1,pkg2 <entrypoint>`{maybe_install_example}):\n      {packages}", crate::colors::yellow("warning"));
+  }
+
   setup_cache.save();
   drop(single_process_lock);
   drop(pb_clear_guard);