feat(npm): functionality to support child_process.fork (#15891)

cli/npm/registry.rs

@@ -92,7 +92,7 @@ impl NpmPackageVersionInfo {
   }
 }
 
-#[derive(Debug, Default, Deserialize, Serialize, Clone)]
+#[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct NpmPackageVersionDistInfo {
   /// URL to the tarball.
   pub tarball: String,

cli/npm/resolution.rs

@@ -11,6 +11,8 @@ use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::futures;
 use deno_core::parking_lot::RwLock;
+use serde::Deserialize;
+use serde::Serialize;
 
 use super::registry::NpmPackageInfo;
 use super::registry::NpmPackageVersionDistInfo;
@@ -91,7 +93,9 @@ impl std::fmt::Display for NpmPackageReference {
   }
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)]
+#[derive(
+  Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
 pub struct NpmPackageReq {
   pub name: String,
   pub version_req: Option<SpecifierVersionReq>,
@@ -127,7 +131,9 @@ impl NpmVersionMatcher for NpmPackageReq {
   }
 }
 
-#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash)]
+#[derive(
+  Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
 pub struct NpmPackageId {
   pub name: String,
   pub version: NpmVersion,
@@ -150,7 +156,7 @@ impl std::fmt::Display for NpmPackageId {
   }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct NpmResolutionPackage {
   pub id: NpmPackageId,
   pub dist: NpmPackageVersionDistInfo,
@@ -159,13 +165,54 @@ pub struct NpmResolutionPackage {
   pub dependencies: HashMap<String, NpmPackageId>,
 }
 
-#[derive(Debug, Clone, Default)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct NpmResolutionSnapshot {
+  #[serde(with = "map_to_vec")]
   package_reqs: HashMap<NpmPackageReq, NpmVersion>,
   packages_by_name: HashMap<String, Vec<NpmVersion>>,
+  #[serde(with = "map_to_vec")]
   packages: HashMap<NpmPackageId, NpmResolutionPackage>,
 }
 
+// This is done so the maps with non-string keys get serialized and deserialized as vectors.
+// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792
+mod map_to_vec {
+  use std::collections::HashMap;
+
+  use serde::de::Deserialize;
+  use serde::de::Deserializer;
+  use serde::ser::Serializer;
+  use serde::Serialize;
+
+  pub fn serialize<S, K: Serialize, V: Serialize>(
+    map: &HashMap<K, V>,
+    serializer: S,
+  ) -> Result<S::Ok, S::Error>
+  where
+    S: Serializer,
+  {
+    serializer.collect_seq(map.iter())
+  }
+
+  pub fn deserialize<
+    'de,
+    D,
+    K: Deserialize<'de> + Eq + std::hash::Hash,
+    V: Deserialize<'de>,
+  >(
+    deserializer: D,
+  ) -> Result<HashMap<K, V>, D::Error>
+  where
+    D: Deserializer<'de>,
+  {
+    let mut map = HashMap::new();
+    for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? {
+      map.insert(key, value);
+    }
+    Ok(map)
+  }
+}
+
 impl NpmResolutionSnapshot {
   /// Resolve a node package from a deno module.
   pub fn resolve_package_from_deno_module(
@@ -292,10 +339,13 @@ impl std::fmt::Debug for NpmResolution {
 }
 
 impl NpmResolution {
-  pub fn new(api: NpmRegistryApi) -> Self {
+  pub fn new(
+    api: NpmRegistryApi,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
+  ) -> Self {
     Self {
       api,
-      snapshot: Default::default(),
+      snapshot: RwLock::new(initial_snapshot.unwrap_or_default()),
       update_sempahore: tokio::sync::Semaphore::new(1),
     }
   }

cli/npm/resolvers/common.rs

@@ -10,6 +10,7 @@ use deno_core::futures;
 use deno_core::futures::future::BoxFuture;
 use deno_core::url::Url;
 
+use crate::npm::resolution::NpmResolutionSnapshot;
 use crate::npm::NpmCache;
 use crate::npm::NpmPackageReq;
 use crate::npm::NpmResolutionPackage;
@@ -39,6 +40,8 @@ pub trait InnerNpmPackageResolver: Send + Sync {
   ) -> BoxFuture<'static, Result<(), AnyError>>;
 
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError>;
+
+  fn snapshot(&self) -> NpmResolutionSnapshot;
 }
 
 /// Caches all the packages in parallel.

cli/npm/resolvers/global.rs

@@ -13,6 +13,7 @@ use deno_core::futures::FutureExt;
 use deno_core::url::Url;
 
 use crate::npm::resolution::NpmResolution;
+use crate::npm::resolution::NpmResolutionSnapshot;
 use crate::npm::resolvers::common::cache_packages;
 use crate::npm::NpmCache;
 use crate::npm::NpmPackageId;
@@ -31,9 +32,13 @@ pub struct GlobalNpmPackageResolver {
 }
 
 impl GlobalNpmPackageResolver {
-  pub fn new(cache: NpmCache, api: NpmRegistryApi) -> Self {
+  pub fn new(
+    cache: NpmCache,
+    api: NpmRegistryApi,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
+  ) -> Self {
     let registry_url = api.base_url().to_owned();
-    let resolution = Arc::new(NpmResolution::new(api));
+    let resolution = Arc::new(NpmResolution::new(api, initial_snapshot));
 
     Self {
       cache,
@@ -105,4 +110,8 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
     let registry_path = self.cache.registry_folder(&self.registry_url);
     ensure_registry_read_permission(&registry_path, path)
   }
+
+  fn snapshot(&self) -> NpmResolutionSnapshot {
+    self.resolution.snapshot()
+  }
 }

cli/npm/resolvers/local.rs

@@ -47,9 +47,10 @@ impl LocalNpmPackageResolver {
     cache: NpmCache,
     api: NpmRegistryApi,
     node_modules_folder: PathBuf,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
   ) -> Self {
     let registry_url = api.base_url().to_owned();
-    let resolution = Arc::new(NpmResolution::new(api));
+    let resolution = Arc::new(NpmResolution::new(api, initial_snapshot));
 
     Self {
       cache,
@@ -180,6 +181,10 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver {
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
     ensure_registry_read_permission(&self.root_node_modules_path, path)
   }
+
+  fn snapshot(&self) -> NpmResolutionSnapshot {
+    self.resolution.snapshot()
+  }
 }
 
 /// Creates a pnpm style folder structure.

cli/npm/resolvers/mod.rs

@@ -8,9 +8,13 @@ use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::bail;
 use deno_core::error::custom_error;
 use deno_core::error::AnyError;
+use deno_core::serde_json;
 use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use global::GlobalNpmPackageResolver;
+use once_cell::sync::Lazy;
+use serde::Deserialize;
+use serde::Serialize;
 
 use std::path::Path;
 use std::path::PathBuf;
@@ -20,15 +24,50 @@ use crate::fs_util;
 
 use self::common::InnerNpmPackageResolver;
 use self::local::LocalNpmPackageResolver;
+use super::resolution::NpmResolutionSnapshot;
 use super::NpmCache;
 use super::NpmPackageReq;
 use super::NpmRegistryApi;
 
+const RESOLUTION_STATE_ENV_VAR_NAME: &str =
+  "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE";
+
+static IS_NPM_MAIN: Lazy<bool> =
+  Lazy::new(|| std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).is_ok());
+
+/// State provided to the process via an environment variable.
+#[derive(Debug, Serialize, Deserialize)]
+struct NpmProcessState {
+  snapshot: NpmResolutionSnapshot,
+  local_node_modules_path: Option<String>,
+}
+
+impl NpmProcessState {
+  pub fn was_set() -> bool {
+    *IS_NPM_MAIN
+  }
+
+  pub fn take() -> Option<NpmProcessState> {
+    // initialize the lazy before we remove the env var below
+    if !Self::was_set() {
+      return None;
+    }
+
+    let state = std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).ok()?;
+    let state = serde_json::from_str(&state).ok()?;
+    // remove the environment variable so that sub processes
+    // that are spawned do not also use this.
+    std::env::remove_var(RESOLUTION_STATE_ENV_VAR_NAME);
+    Some(state)
+  }
+}
+
 #[derive(Clone)]
 pub struct NpmPackageResolver {
   unstable: bool,
   no_npm: bool,
   inner: Arc<dyn InnerNpmPackageResolver>,
+  local_node_modules_path: Option<PathBuf>,
 }
 
 impl NpmPackageResolver {
@@ -39,19 +78,30 @@ impl NpmPackageResolver {
     no_npm: bool,
     local_node_modules_path: Option<PathBuf>,
   ) -> Self {
-    let inner: Arc<dyn InnerNpmPackageResolver> = match local_node_modules_path
+    let process_npm_state = NpmProcessState::take();
+    let local_node_modules_path = local_node_modules_path.or_else(|| {
+      process_npm_state
+        .as_ref()
+        .and_then(|s| s.local_node_modules_path.as_ref().map(PathBuf::from))
+    });
+    let maybe_snapshot = process_npm_state.map(|s| s.snapshot);
+    let inner: Arc<dyn InnerNpmPackageResolver> = match &local_node_modules_path
     {
       Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
         cache,
         api,
-        node_modules_folder,
+        node_modules_folder.clone(),
+        maybe_snapshot,
       )),
-      None => Arc::new(GlobalNpmPackageResolver::new(cache, api)),
+      None => {
+        Arc::new(GlobalNpmPackageResolver::new(cache, api, maybe_snapshot))
+      }
     };
     Self {
       unstable,
       no_npm,
       inner,
+      local_node_modules_path,
     }
   }
 
@@ -137,6 +187,26 @@ impl NpmPackageResolver {
 
     self.inner.add_package_reqs(packages).await
   }
+
+  // If the main module should be treated as being in an npm package.
+  // This is triggered via a secret environment variable which is used
+  // for functionality like child_process.fork. Users should NOT depend
+  // on this functionality.
+  pub fn is_npm_main(&self) -> bool {
+    NpmProcessState::was_set()
+  }
+
+  /// Gets the state of npm for the process.
+  pub fn get_npm_process_state(&self) -> String {
+    serde_json::to_string(&NpmProcessState {
+      snapshot: self.inner.snapshot(),
+      local_node_modules_path: self
+        .local_node_modules_path
+        .as_ref()
+        .map(|p| p.to_string_lossy().to_string()),
+    })
+    .unwrap()
+  }
 }
 
 impl RequireNpmResolver for NpmPackageResolver {

cli/npm/semver/mod.rs

@@ -6,6 +6,8 @@ use std::fmt;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use monch::*;
+use serde::Deserialize;
+use serde::Serialize;
 
 use crate::npm::resolution::NpmVersionMatcher;
 
@@ -25,7 +27,9 @@ mod specifier;
 // A lot of the below is a re-implementation of parts of https://github.com/npm/node-semver
 // which is Copyright (c) Isaac Z. Schlueter and Contributors (ISC License)
 
-#[derive(Clone, Debug, PartialEq, Eq, Default, Hash)]
+#[derive(
+  Clone, Debug, PartialEq, Eq, Default, Hash, Serialize, Deserialize,
+)]
 pub struct NpmVersion {
   pub major: u64,
   pub minor: u64,

cli/npm/semver/range.rs

@@ -2,6 +2,9 @@
 
 use std::cmp::Ordering;
 
+use serde::Deserialize;
+use serde::Serialize;
+
 use super::NpmVersion;
 
 /// Collection of ranges.
@@ -14,7 +17,7 @@ impl VersionRangeSet {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub enum RangeBound {
   Version(VersionBound),
   Unbounded, // matches everything
@@ -91,13 +94,13 @@ impl RangeBound {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub enum VersionBoundKind {
   Inclusive,
   Exclusive,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct VersionBound {
   pub kind: VersionBoundKind,
   pub version: NpmVersion,
@@ -109,7 +112,7 @@ impl VersionBound {
   }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct VersionRange {
   pub start: RangeBound,
   pub end: RangeBound,

cli/npm/semver/specifier.rs

@@ -3,6 +3,8 @@
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use monch::*;
+use serde::Deserialize;
+use serde::Serialize;
 
 use super::errors::with_failure_handling;
 use super::range::Partial;
@@ -11,7 +13,7 @@ use super::range::XRange;
 use super::NpmVersion;
 
 /// Version requirement found in npm specifiers.
-#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct SpecifierVersionReq {
   raw_text: String,
   range: VersionRange,