feat(npm): add flag for creating and resolving npm packages to a local node_modules folder (#15971)

cli/npm/resolvers/local.rs

@@ -0,0 +1,321 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+//! Code for local node_modules resolution.
+
+use std::collections::HashSet;
+use std::collections::VecDeque;
+use std::fs;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::futures::future::BoxFuture;
+use deno_core::futures::FutureExt;
+use deno_core::url::Url;
+
+use crate::fs_util;
+use crate::npm::resolution::NpmResolution;
+use crate::npm::resolution::NpmResolutionSnapshot;
+use crate::npm::NpmCache;
+use crate::npm::NpmPackageId;
+use crate::npm::NpmPackageReq;
+use crate::npm::NpmRegistryApi;
+
+use super::common::cache_packages;
+use super::common::ensure_registry_read_permission;
+use super::common::InnerNpmPackageResolver;
+
+/// Resolver that creates a local node_modules directory
+/// and resolves packages from it.
+#[derive(Debug, Clone)]
+pub struct LocalNpmPackageResolver {
+  cache: NpmCache,
+  resolution: Arc<NpmResolution>,
+  registry_url: Url,
+  root_node_modules_path: PathBuf,
+  root_node_modules_specifier: ModuleSpecifier,
+}
+
+impl LocalNpmPackageResolver {
+  pub fn new(
+    cache: NpmCache,
+    api: NpmRegistryApi,
+    node_modules_folder: PathBuf,
+  ) -> Self {
+    let registry_url = api.base_url().to_owned();
+    let resolution = Arc::new(NpmResolution::new(api));
+
+    Self {
+      cache,
+      resolution,
+      registry_url,
+      root_node_modules_specifier: ModuleSpecifier::from_directory_path(
+        &node_modules_folder,
+      )
+      .unwrap(),
+      root_node_modules_path: node_modules_folder,
+    }
+  }
+
+  fn resolve_package_root(&self, path: &Path) -> PathBuf {
+    let mut last_found = path;
+    loop {
+      let parent = last_found.parent().unwrap();
+      if parent.file_name().unwrap() == "node_modules" {
+        return last_found.to_path_buf();
+      } else {
+        last_found = parent;
+      }
+    }
+  }
+
+  fn resolve_folder_for_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    match self.maybe_resolve_folder_for_specifier(specifier) {
+      Some(path) => Ok(path),
+      None => bail!("could not find npm package for '{}'", specifier),
+    }
+  }
+
+  fn maybe_resolve_folder_for_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Option<PathBuf> {
+    let relative_url =
+      self.root_node_modules_specifier.make_relative(specifier)?;
+    if relative_url.starts_with("../") {
+      return None;
+    }
+    // it's within the directory, so use it
+    specifier.to_file_path().ok()
+  }
+}
+
+impl InnerNpmPackageResolver for LocalNpmPackageResolver {
+  fn resolve_package_folder_from_deno_module(
+    &self,
+    pkg_req: &NpmPackageReq,
+  ) -> Result<PathBuf, AnyError> {
+    let resolved_package =
+      self.resolution.resolve_package_from_deno_module(pkg_req)?;
+
+    // it might be at the full path if there are duplicate names
+    let fully_resolved_folder_path = join_package_name(
+      &self.root_node_modules_path,
+      &resolved_package.id.to_string(),
+    );
+    Ok(if fully_resolved_folder_path.exists() {
+      fully_resolved_folder_path
+    } else {
+      join_package_name(&self.root_node_modules_path, &resolved_package.id.name)
+    })
+  }
+
+  fn resolve_package_folder_from_package(
+    &self,
+    name: &str,
+    referrer: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    let local_path = self.resolve_folder_for_specifier(referrer)?;
+    let package_root_path = self.resolve_package_root(&local_path);
+    let mut current_folder = package_root_path.as_path();
+    loop {
+      current_folder = get_next_node_modules_ancestor(current_folder);
+      let sub_dir = join_package_name(current_folder, name);
+      if sub_dir.is_dir() {
+        return Ok(sub_dir);
+      }
+      if current_folder == self.root_node_modules_path {
+        bail!(
+          "could not find package '{}' from referrer '{}'.",
+          name,
+          referrer
+        );
+      }
+    }
+  }
+
+  fn resolve_package_folder_from_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    let local_path = self.resolve_folder_for_specifier(specifier)?;
+    let package_root_path = self.resolve_package_root(&local_path);
+    Ok(package_root_path)
+  }
+
+  fn has_packages(&self) -> bool {
+    self.resolution.has_packages()
+  }
+
+  fn add_package_reqs(
+    &self,
+    packages: Vec<NpmPackageReq>,
+  ) -> BoxFuture<'static, Result<(), AnyError>> {
+    let resolver = self.clone();
+    async move {
+      resolver.resolution.add_package_reqs(packages).await?;
+      cache_packages(
+        resolver.resolution.all_packages(),
+        &resolver.cache,
+        &resolver.registry_url,
+      )
+      .await?;
+
+      sync_resolution_with_fs(
+        &resolver.resolution.snapshot(),
+        &resolver.cache,
+        &resolver.registry_url,
+        &resolver.root_node_modules_path,
+      )?;
+
+      Ok(())
+    }
+    .boxed()
+  }
+
+  fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
+    ensure_registry_read_permission(&self.root_node_modules_path, path)
+  }
+}
+
+/// Creates a pnpm style folder structure.
+fn sync_resolution_with_fs(
+  snapshot: &NpmResolutionSnapshot,
+  cache: &NpmCache,
+  registry_url: &Url,
+  root_node_modules_dir_path: &Path,
+) -> Result<(), AnyError> {
+  fn get_package_folder_name(package_id: &NpmPackageId) -> String {
+    package_id.to_string().replace('/', "+")
+  }
+
+  let deno_local_registry_dir = root_node_modules_dir_path.join(".deno");
+  fs::create_dir_all(&deno_local_registry_dir).with_context(|| {
+    format!("Creating '{}'", deno_local_registry_dir.display())
+  })?;
+
+  // 1. Write all the packages out the .deno directory.
+  //
+  // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to
+  // node_modules/.deno/<package_id>/node_modules/<package_name>
+  let all_packages = snapshot.all_packages();
+  for package in &all_packages {
+    let folder_name = get_package_folder_name(&package.id);
+    let folder_path = deno_local_registry_dir.join(&folder_name);
+    let initialized_file = folder_path.join("deno_initialized");
+    if !initialized_file.exists() {
+      let sub_node_modules = folder_path.join("node_modules");
+      let package_path = join_package_name(&sub_node_modules, &package.id.name);
+      fs::create_dir_all(&package_path)
+        .with_context(|| format!("Creating '{}'", folder_path.display()))?;
+      let cache_folder = cache.package_folder(&package.id, registry_url);
+      // for now copy, but in the future consider hard linking
+      fs_util::copy_dir_recursive(&cache_folder, &package_path)?;
+      // write out a file that indicates this folder has been initialized
+      fs::write(initialized_file, "")?;
+    }
+  }
+
+  // 2. Symlink all the dependencies into the .deno directory.
+  //
+  // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to
+  // node_modules/.deno/<dep_id>/node_modules/<dep_package_name>
+  for package in &all_packages {
+    let sub_node_modules = deno_local_registry_dir
+      .join(&get_package_folder_name(&package.id))
+      .join("node_modules");
+    for (name, dep_id) in &package.dependencies {
+      let dep_folder_name = get_package_folder_name(dep_id);
+      let dep_folder_path = join_package_name(
+        &deno_local_registry_dir
+          .join(dep_folder_name)
+          .join("node_modules"),
+        &dep_id.name,
+      );
+      symlink_package_dir(
+        &dep_folder_path,
+        &join_package_name(&sub_node_modules, name),
+      )?;
+    }
+  }
+
+  // 3. Create all the packages in the node_modules folder, which are symlinks.
+  //
+  // Symlink node_modules/<package_name> to
+  // node_modules/.deno/<package_id>/node_modules/<package_name>
+  let mut found_names = HashSet::new();
+  let mut pending_packages = VecDeque::new();
+  pending_packages.extend(
+    snapshot
+      .top_level_packages()
+      .into_iter()
+      .map(|id| (id, true)),
+  );
+  while let Some((package_id, is_top_level)) = pending_packages.pop_front() {
+    let root_folder_name = if found_names.insert(package_id.name.clone()) {
+      package_id.name.clone()
+    } else if is_top_level {
+      package_id.to_string()
+    } else {
+      continue; // skip, already handled
+    };
+    let local_registry_package_path = deno_local_registry_dir
+      .join(&get_package_folder_name(&package_id))
+      .join("node_modules")
+      .join(&package_id.name);
+
+    symlink_package_dir(
+      &local_registry_package_path,
+      &join_package_name(root_node_modules_dir_path, &root_folder_name),
+    )?;
+    if let Some(package) = snapshot.package_from_id(&package_id) {
+      for id in package.dependencies.values() {
+        pending_packages.push_back((id.clone(), false));
+      }
+    }
+  }
+
+  Ok(())
+}
+
+fn symlink_package_dir(
+  old_path: &Path,
+  new_path: &Path,
+) -> Result<(), AnyError> {
+  let new_parent = new_path.parent().unwrap();
+  if new_parent.file_name().unwrap() != "node_modules" {
+    // create the parent folder that will contain the symlink
+    fs::create_dir_all(new_parent)
+      .with_context(|| format!("Creating '{}'", new_parent.display()))?;
+  }
+
+  // need to delete the previous symlink before creating a new one
+  let _ignore = fs::remove_dir_all(new_path);
+  fs_util::symlink_dir(old_path, new_path)
+}
+
+fn join_package_name(path: &Path, package_name: &str) -> PathBuf {
+  let mut path = path.to_path_buf();
+  // ensure backslashes are used on windows
+  for part in package_name.split('/') {
+    path = path.join(part);
+  }
+  path
+}
+
+fn get_next_node_modules_ancestor(mut path: &Path) -> &Path {
+  loop {
+    path = path.parent().unwrap();
+    let file_name = path.file_name().unwrap().to_string_lossy();
+    if file_name == "node_modules" {
+      return path;
+    }
+  }
+}