feat(compile): unstable npm and node specifier support (#19005)

This is the initial support for npm and node specifiers in `deno
compile`. The npm packages are included in the binary and read from it via
a virtual file system. This also supports the `--node-modules-dir` flag,
dependencies specified in a package.json, and npm binary commands (ex.
`deno compile --unstable npm:cowsay`)

Closes #16632

cli/npm/installer.rs

@@ -10,7 +10,7 @@ use deno_npm::registry::NpmRegistryApi;
 use deno_npm::registry::NpmRegistryPackageInfoLoadError;
 use deno_semver::npm::NpmPackageReq;
 
-use crate::args::package_json::PackageJsonDeps;
+use crate::args::PackageJsonDepsProvider;
 use crate::util::sync::AtomicFlag;
 
 use super::CliNpmRegistryApi;
@@ -18,23 +18,13 @@ use super::NpmResolution;
 
 #[derive(Debug)]
 struct PackageJsonDepsInstallerInner {
+  deps_provider: Arc<PackageJsonDepsProvider>,
   has_installed_flag: AtomicFlag,
   npm_registry_api: Arc<CliNpmRegistryApi>,
   npm_resolution: Arc<NpmResolution>,
-  package_deps: PackageJsonDeps,
 }
 
 impl PackageJsonDepsInstallerInner {
-  pub fn reqs(&self) -> Vec<&NpmPackageReq> {
-    let mut package_reqs = self
-      .package_deps
-      .values()
-      .filter_map(|r| r.as_ref().ok())
-      .collect::<Vec<_>>();
-    package_reqs.sort(); // deterministic resolution
-    package_reqs
-  }
-
   pub fn reqs_with_info_futures(
     &self,
   ) -> FuturesOrdered<
@@ -45,7 +35,7 @@ impl PackageJsonDepsInstallerInner {
       >,
     >,
   > {
-    let package_reqs = self.reqs();
+    let package_reqs = self.deps_provider.reqs();
 
     FuturesOrdered::from_iter(package_reqs.into_iter().map(|req| {
       let api = self.npm_registry_api.clone();
@@ -63,22 +53,18 @@ pub struct PackageJsonDepsInstaller(Option<PackageJsonDepsInstallerInner>);
 
 impl PackageJsonDepsInstaller {
   pub fn new(
+    deps_provider: Arc<PackageJsonDepsProvider>,
     npm_registry_api: Arc<CliNpmRegistryApi>,
     npm_resolution: Arc<NpmResolution>,
-    deps: Option<PackageJsonDeps>,
   ) -> Self {
-    Self(deps.map(|package_deps| PackageJsonDepsInstallerInner {
+    Self(Some(PackageJsonDepsInstallerInner {
+      deps_provider,
       has_installed_flag: Default::default(),
       npm_registry_api,
       npm_resolution,
-      package_deps,
     }))
   }
 
-  pub fn package_deps(&self) -> Option<&PackageJsonDeps> {
-    self.0.as_ref().map(|inner| &inner.package_deps)
-  }
-
   /// Installs the top level dependencies in the package.json file
   /// without going through and resolving the descendant dependencies yet.
   pub async fn ensure_top_level_install(&self) -> Result<(), AnyError> {

cli/npm/resolution.rs

@@ -237,6 +237,10 @@ impl NpmResolution {
     Ok(nv)
   }
 
+  pub fn all_packages(&self) -> Vec<NpmResolutionPackage> {
+    self.snapshot.read().all_packages()
+  }
+
   pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned {
     self.snapshot.read().all_packages_partitioned()
   }

cli/npm/resolvers/common.rs

@@ -12,6 +12,7 @@ use deno_core::futures;
 use deno_core::url::Url;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
+use deno_runtime::deno_fs::FileSystem;
 use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::deno_node::NodeResolutionMode;
 
@@ -90,6 +91,7 @@ pub async fn cache_packages(
 }
 
 pub fn ensure_registry_read_permission(
+  fs: &Arc<dyn FileSystem>,
   permissions: &dyn NodePermissions,
   registry_path: &Path,
   path: &Path,
@@ -101,8 +103,8 @@ pub fn ensure_registry_read_permission(
       .all(|c| !matches!(c, std::path::Component::ParentDir))
   {
     // todo(dsherret): cache this?
-    if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
-      match std::fs::canonicalize(path) {
+    if let Ok(registry_path) = fs.realpath_sync(registry_path) {
+      match fs.realpath_sync(path) {
         Ok(path) if path.starts_with(registry_path) => {
           return Ok(());
         }

cli/npm/resolvers/global.rs

@@ -14,6 +14,7 @@ use deno_npm::resolution::PackageNotFoundFromReferrerError;
 use deno_npm::NpmPackageCacheFolderId;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
+use deno_runtime::deno_fs::FileSystem;
 use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::deno_node::NodeResolutionMode;
 
@@ -28,6 +29,7 @@ use super::common::NpmPackageFsResolver;
 /// Resolves packages from the global npm cache.
 #[derive(Debug)]
 pub struct GlobalNpmPackageResolver {
+  fs: Arc<dyn FileSystem>,
   cache: Arc<NpmCache>,
   resolution: Arc<NpmResolution>,
   registry_url: Url,
@@ -35,11 +37,13 @@ pub struct GlobalNpmPackageResolver {
 
 impl GlobalNpmPackageResolver {
   pub fn new(
+    fs: Arc<dyn FileSystem>,
     cache: Arc<NpmCache>,
     registry_url: Url,
     resolution: Arc<NpmResolution>,
   ) -> Self {
     Self {
+      fs,
       cache,
       resolution,
       registry_url,
@@ -130,7 +134,7 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver {
     path: &Path,
   ) -> Result<(), AnyError> {
     let registry_path = self.cache.registry_folder(&self.registry_url);
-    ensure_registry_read_permission(permissions, &registry_path, path)
+    ensure_registry_read_permission(&self.fs, permissions, &registry_path, path)
   }
 }
 

cli/npm/resolvers/local.rs

@@ -154,7 +154,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
     loop {
       current_folder = get_next_node_modules_ancestor(current_folder);
       let sub_dir = join_package_name(current_folder, name);
-      if sub_dir.is_dir() {
+      if self.fs.is_dir(&sub_dir) {
         // if doing types resolution, only resolve the package if it specifies a types property
         if mode.is_types() && !name.starts_with("@types/") {
           let package_json = PackageJson::load_skip_read_permission(
@@ -173,7 +173,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
       if mode.is_types() && !name.starts_with("@types/") {
         let sub_dir =
           join_package_name(current_folder, &types_package_name(name));
-        if sub_dir.is_dir() {
+        if self.fs.is_dir(&sub_dir) {
           return Ok(sub_dir);
         }
       }
@@ -214,6 +214,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
     path: &Path,
   ) -> Result<(), AnyError> {
     ensure_registry_read_permission(
+      &self.fs,
       permissions,
       &self.root_node_modules_path,
       path,

cli/npm/resolvers/mod.rs

@@ -18,7 +18,7 @@ use deno_npm::resolution::NpmResolutionSnapshot;
 use deno_npm::resolution::PackageReqNotFoundError;
 use deno_npm::resolution::SerializedNpmResolutionSnapshot;
 use deno_npm::NpmPackageId;
-use deno_runtime::deno_fs;
+use deno_runtime::deno_fs::FileSystem;
 use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::deno_node::NpmResolver;
@@ -32,7 +32,7 @@ use serde::Deserialize;
 use serde::Serialize;
 
 use crate::args::Lockfile;
-use crate::util::fs::canonicalize_path_maybe_not_exists;
+use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
 use crate::util::progress_bar::ProgressBar;
 
 use self::common::NpmPackageFsResolver;
@@ -49,6 +49,7 @@ pub struct NpmProcessState {
 
 /// Brings together the npm resolution with the file system.
 pub struct CliNpmResolver {
+  fs: Arc<dyn FileSystem>,
   fs_resolver: Arc<dyn NpmPackageFsResolver>,
   resolution: Arc<NpmResolution>,
   maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
@@ -57,6 +58,7 @@ pub struct CliNpmResolver {
 impl std::fmt::Debug for CliNpmResolver {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     f.debug_struct("NpmPackageResolver")
+      .field("fs", &"<omitted>")
       .field("fs_resolver", &"<omitted>")
       .field("resolution", &"<omitted>")
       .field("maybe_lockfile", &"<omitted>")
@@ -66,11 +68,13 @@ impl std::fmt::Debug for CliNpmResolver {
 
 impl CliNpmResolver {
   pub fn new(
+    fs: Arc<dyn FileSystem>,
     resolution: Arc<NpmResolution>,
     fs_resolver: Arc<dyn NpmPackageFsResolver>,
     maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
   ) -> Self {
     Self {
+      fs,
       fs_resolver,
       resolution,
       maybe_lockfile,
@@ -81,6 +85,10 @@ impl CliNpmResolver {
     self.fs_resolver.root_dir_url()
   }
 
+  pub fn node_modules_path(&self) -> Option<PathBuf> {
+    self.fs_resolver.node_modules_path()
+  }
+
   pub fn resolve_pkg_id_from_pkg_req(
     &self,
     req: &NpmPackageReq,
@@ -88,12 +96,17 @@ impl CliNpmResolver {
     self.resolution.resolve_pkg_id_from_pkg_req(req)
   }
 
-  fn resolve_pkg_folder_from_deno_module_at_pkg_id(
+  pub fn resolve_pkg_folder_from_pkg_id(
     &self,
     pkg_id: &NpmPackageId,
   ) -> Result<PathBuf, AnyError> {
     let path = self.fs_resolver.package_folder(pkg_id)?;
-    let path = canonicalize_path_maybe_not_exists(&path)?;
+    let path = canonicalize_path_maybe_not_exists_with_fs(&path, |path| {
+      self
+        .fs
+        .realpath_sync(path)
+        .map_err(|err| err.into_io_error())
+    })?;
     log::debug!(
       "Resolved package folder of {} to {}",
       pkg_id.as_serialized(),
@@ -237,7 +250,7 @@ impl NpmResolver for CliNpmResolver {
     pkg_nv: &NpmPackageNv,
   ) -> Result<PathBuf, AnyError> {
     let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(pkg_nv)?;
-    self.resolve_pkg_folder_from_deno_module_at_pkg_id(&pkg_id)
+    self.resolve_pkg_folder_from_pkg_id(&pkg_id)
   }
 
   fn resolve_pkg_id_from_pkg_req(
@@ -270,7 +283,7 @@ impl NpmResolver for CliNpmResolver {
 }
 
 pub fn create_npm_fs_resolver(
-  fs: Arc<dyn deno_fs::FileSystem>,
+  fs: Arc<dyn FileSystem>,
   cache: Arc<NpmCache>,
   progress_bar: &ProgressBar,
   registry_url: Url,
@@ -287,6 +300,7 @@ pub fn create_npm_fs_resolver(
       resolution,
     )),
     None => Arc::new(GlobalNpmPackageResolver::new(
+      fs,
       cache,
       registry_url,
       resolution,