feat(unstable/npm): support peer dependencies (#16561)

This adds support for peer dependencies in npm packages.

1. If not found higher in the tree (ancestor and ancestor siblings),
peer dependencies are resolved like a dependency similar to npm 7.
2. Optional peer dependencies are only resolved if found higher in the
tree.
3. This creates "copy packages" or duplicates of a package when a
package has different resolution due to peer dependency resolution—see
https://pnpm.io/how-peers-are-resolved. Unlike pnpm though, duplicates
of packages will have `_1`, `_2`, etc. added to the end of the package
version in the directory in order to minimize the chance of hitting the
max file path limit on Windows. This is done for both the local
"node_modules" directory and also the global npm cache. The files are
hard linked in this case to reduce hard drive space.

This is a first pass and the code is definitely more inefficient than it
could be.

Closes #15823

cli/npm/resolution/mod.rs

@@ -0,0 +1,676 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::collections::HashSet;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::generic_error;
+use deno_core::error::AnyError;
+use deno_core::futures;
+use deno_core::parking_lot::RwLock;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::lockfile::Lockfile;
+
+use self::graph::GraphDependencyResolver;
+use self::snapshot::NpmPackagesPartitioned;
+
+use super::cache::should_sync_download;
+use super::cache::NpmPackageCacheFolderId;
+use super::registry::NpmPackageVersionDistInfo;
+use super::registry::RealNpmRegistryApi;
+use super::semver::NpmVersion;
+use super::semver::SpecifierVersionReq;
+use super::NpmRegistryApi;
+
+mod graph;
+mod snapshot;
+
+use graph::Graph;
+pub use snapshot::NpmResolutionSnapshot;
+
+/// The version matcher used for npm schemed urls is more strict than
+/// the one used by npm packages and so we represent either via a trait.
+pub trait NpmVersionMatcher {
+  fn tag(&self) -> Option<&str>;
+  fn matches(&self, version: &NpmVersion) -> bool;
+  fn version_text(&self) -> String;
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct NpmPackageReference {
+  pub req: NpmPackageReq,
+  pub sub_path: Option<String>,
+}
+
+impl NpmPackageReference {
+  pub fn from_specifier(
+    specifier: &ModuleSpecifier,
+  ) -> Result<NpmPackageReference, AnyError> {
+    Self::from_str(specifier.as_str())
+  }
+
+  pub fn from_str(specifier: &str) -> Result<NpmPackageReference, AnyError> {
+    let specifier = match specifier.strip_prefix("npm:") {
+      Some(s) => s,
+      None => {
+        bail!("Not an npm specifier: {}", specifier);
+      }
+    };
+    let parts = specifier.split('/').collect::<Vec<_>>();
+    let name_part_len = if specifier.starts_with('@') { 2 } else { 1 };
+    if parts.len() < name_part_len {
+      return Err(generic_error(format!("Not a valid package: {}", specifier)));
+    }
+    let name_parts = &parts[0..name_part_len];
+    let last_name_part = &name_parts[name_part_len - 1];
+    let (name, version_req) = if let Some(at_index) = last_name_part.rfind('@')
+    {
+      let version = &last_name_part[at_index + 1..];
+      let last_name_part = &last_name_part[..at_index];
+      let version_req = SpecifierVersionReq::parse(version)
+        .with_context(|| "Invalid version requirement.")?;
+      let name = if name_part_len == 1 {
+        last_name_part.to_string()
+      } else {
+        format!("{}/{}", name_parts[0], last_name_part)
+      };
+      (name, Some(version_req))
+    } else {
+      (name_parts.join("/"), None)
+    };
+    let sub_path = if parts.len() == name_parts.len() {
+      None
+    } else {
+      Some(parts[name_part_len..].join("/"))
+    };
+
+    if let Some(sub_path) = &sub_path {
+      if let Some(at_index) = sub_path.rfind('@') {
+        let (new_sub_path, version) = sub_path.split_at(at_index);
+        let msg = format!(
+          "Invalid package specifier 'npm:{}/{}'. Did you mean to write 'npm:{}{}/{}'?",
+          name, sub_path, name, version, new_sub_path
+        );
+        return Err(generic_error(msg));
+      }
+    }
+
+    Ok(NpmPackageReference {
+      req: NpmPackageReq { name, version_req },
+      sub_path,
+    })
+  }
+}
+
+impl std::fmt::Display for NpmPackageReference {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    if let Some(sub_path) = &self.sub_path {
+      write!(f, "npm:{}/{}", self.req, sub_path)
+    } else {
+      write!(f, "npm:{}", self.req)
+    }
+  }
+}
+
+#[derive(
+  Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
+pub struct NpmPackageReq {
+  pub name: String,
+  pub version_req: Option<SpecifierVersionReq>,
+}
+
+impl std::fmt::Display for NpmPackageReq {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match &self.version_req {
+      Some(req) => write!(f, "{}@{}", self.name, req),
+      None => write!(f, "{}", self.name),
+    }
+  }
+}
+
+impl NpmPackageReq {
+  pub fn from_str(text: &str) -> Result<Self, AnyError> {
+    // probably should do something more targetted in the future
+    let reference = NpmPackageReference::from_str(&format!("npm:{}", text))?;
+    Ok(reference.req)
+  }
+}
+
+impl NpmVersionMatcher for NpmPackageReq {
+  fn tag(&self) -> Option<&str> {
+    match &self.version_req {
+      Some(version_req) => version_req.tag(),
+      None => Some("latest"),
+    }
+  }
+
+  fn matches(&self, version: &NpmVersion) -> bool {
+    match self.version_req.as_ref() {
+      Some(req) => {
+        assert_eq!(self.tag(), None);
+        match req.range() {
+          Some(range) => range.satisfies(version),
+          None => false,
+        }
+      }
+      None => version.pre.is_empty(),
+    }
+  }
+
+  fn version_text(&self) -> String {
+    self
+      .version_req
+      .as_ref()
+      .map(|v| format!("{}", v))
+      .unwrap_or_else(|| "non-prerelease".to_string())
+  }
+}
+
+#[derive(
+  Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize,
+)]
+pub struct NpmPackageId {
+  pub name: String,
+  pub version: NpmVersion,
+  pub peer_dependencies: Vec<NpmPackageId>,
+}
+
+impl NpmPackageId {
+  #[allow(unused)]
+  pub fn scope(&self) -> Option<&str> {
+    if self.name.starts_with('@') && self.name.contains('/') {
+      self.name.split('/').next()
+    } else {
+      None
+    }
+  }
+
+  pub fn as_serialized(&self) -> String {
+    self.as_serialized_with_level(0)
+  }
+
+  fn as_serialized_with_level(&self, level: usize) -> String {
+    // WARNING: This should not change because it's used in the lockfile
+    let mut result = format!(
+      "{}@{}",
+      if level == 0 {
+        self.name.to_string()
+      } else {
+        self.name.replace('/', "+")
+      },
+      self.version
+    );
+    for peer in &self.peer_dependencies {
+      // unfortunately we can't do something like `_3` when
+      // this gets deep because npm package names can start
+      // with a number
+      result.push_str(&"_".repeat(level + 1));
+      result.push_str(&peer.as_serialized_with_level(level + 1));
+    }
+    result
+  }
+
+  pub fn from_serialized(id: &str) -> Result<Self, AnyError> {
+    use monch::*;
+
+    fn parse_name(input: &str) -> ParseResult<&str> {
+      if_not_empty(substring(move |input| {
+        for (pos, c) in input.char_indices() {
+          // first character might be a scope, so skip it
+          if pos > 0 && c == '@' {
+            return Ok((&input[pos..], ()));
+          }
+        }
+        ParseError::backtrace()
+      }))(input)
+    }
+
+    fn parse_version(input: &str) -> ParseResult<&str> {
+      if_not_empty(substring(skip_while(|c| c != '_')))(input)
+    }
+
+    fn parse_name_and_version(
+      input: &str,
+    ) -> ParseResult<(String, NpmVersion)> {
+      let (input, name) = parse_name(input)?;
+      let (input, _) = ch('@')(input)?;
+      let at_version_input = input;
+      let (input, version) = parse_version(input)?;
+      match NpmVersion::parse(version) {
+        Ok(version) => Ok((input, (name.to_string(), version))),
+        Err(err) => ParseError::fail(at_version_input, format!("{:#}", err)),
+      }
+    }
+
+    fn parse_level_at_level<'a>(
+      level: usize,
+    ) -> impl Fn(&'a str) -> ParseResult<'a, ()> {
+      fn parse_level(input: &str) -> ParseResult<usize> {
+        let level = input.chars().take_while(|c| *c == '_').count();
+        Ok((&input[level..], level))
+      }
+
+      move |input| {
+        let (input, parsed_level) = parse_level(input)?;
+        if parsed_level == level {
+          Ok((input, ()))
+        } else {
+          ParseError::backtrace()
+        }
+      }
+    }
+
+    fn parse_peers_at_level<'a>(
+      level: usize,
+    ) -> impl Fn(&'a str) -> ParseResult<'a, Vec<NpmPackageId>> {
+      move |mut input| {
+        let mut peers = Vec::new();
+        while let Ok((level_input, _)) = parse_level_at_level(level)(input) {
+          input = level_input;
+          let peer_result = parse_id_at_level(level)(input)?;
+          input = peer_result.0;
+          peers.push(peer_result.1);
+        }
+        Ok((input, peers))
+      }
+    }
+
+    fn parse_id_at_level<'a>(
+      level: usize,
+    ) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageId> {
+      move |input| {
+        let (input, (name, version)) = parse_name_and_version(input)?;
+        let name = if level > 0 {
+          name.replace('+', "/")
+        } else {
+          name
+        };
+        let (input, peer_dependencies) =
+          parse_peers_at_level(level + 1)(input)?;
+        Ok((
+          input,
+          NpmPackageId {
+            name,
+            version,
+            peer_dependencies,
+          },
+        ))
+      }
+    }
+
+    with_failure_handling(parse_id_at_level(0))(id)
+      .with_context(|| format!("Invalid npm package id '{}'.", id))
+  }
+
+  pub fn display(&self) -> String {
+    // Don't implement std::fmt::Display because we don't
+    // want this to be used by accident in certain scenarios.
+    format!("{}@{}", self.name, self.version)
+  }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct NpmResolutionPackage {
+  pub id: NpmPackageId,
+  /// The peer dependency resolution can differ for the same
+  /// package (name and version) depending on where it is in
+  /// the resolution tree. This copy index indicates which
+  /// copy of the package this is.
+  pub copy_index: usize,
+  pub dist: NpmPackageVersionDistInfo,
+  /// Key is what the package refers to the other package as,
+  /// which could be different from the package name.
+  pub dependencies: HashMap<String, NpmPackageId>,
+}
+
+impl NpmResolutionPackage {
+  pub fn get_package_cache_folder_id(&self) -> NpmPackageCacheFolderId {
+    NpmPackageCacheFolderId {
+      name: self.id.name.clone(),
+      version: self.id.version.clone(),
+      copy_index: self.copy_index,
+    }
+  }
+}
+
+pub struct NpmResolution {
+  api: RealNpmRegistryApi,
+  snapshot: RwLock<NpmResolutionSnapshot>,
+  update_sempahore: tokio::sync::Semaphore,
+}
+
+impl std::fmt::Debug for NpmResolution {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    let snapshot = self.snapshot.read();
+    f.debug_struct("NpmResolution")
+      .field("snapshot", &snapshot)
+      .finish()
+  }
+}
+
+impl NpmResolution {
+  pub fn new(
+    api: RealNpmRegistryApi,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
+  ) -> Self {
+    Self {
+      api,
+      snapshot: RwLock::new(initial_snapshot.unwrap_or_default()),
+      update_sempahore: tokio::sync::Semaphore::new(1),
+    }
+  }
+
+  pub async fn add_package_reqs(
+    &self,
+    package_reqs: Vec<NpmPackageReq>,
+  ) -> Result<(), AnyError> {
+    // only allow one thread in here at a time
+    let _permit = self.update_sempahore.acquire().await.unwrap();
+    let snapshot = self.snapshot.read().clone();
+
+    let snapshot = self
+      .add_package_reqs_to_snapshot(package_reqs, snapshot)
+      .await?;
+
+    *self.snapshot.write() = snapshot;
+    Ok(())
+  }
+
+  pub async fn set_package_reqs(
+    &self,
+    package_reqs: HashSet<NpmPackageReq>,
+  ) -> Result<(), AnyError> {
+    // only allow one thread in here at a time
+    let _permit = self.update_sempahore.acquire().await.unwrap();
+    let snapshot = self.snapshot.read().clone();
+
+    let has_removed_package = !snapshot
+      .package_reqs
+      .keys()
+      .all(|req| package_reqs.contains(req));
+    // if any packages were removed, we need to completely recreate the npm resolution snapshot
+    let snapshot = if has_removed_package {
+      NpmResolutionSnapshot::default()
+    } else {
+      snapshot
+    };
+    let snapshot = self
+      .add_package_reqs_to_snapshot(
+        package_reqs.into_iter().collect(),
+        snapshot,
+      )
+      .await?;
+
+    *self.snapshot.write() = snapshot;
+
+    Ok(())
+  }
+
+  async fn add_package_reqs_to_snapshot(
+    &self,
+    mut package_reqs: Vec<NpmPackageReq>,
+    snapshot: NpmResolutionSnapshot,
+  ) -> Result<NpmResolutionSnapshot, AnyError> {
+    // convert the snapshot to a traversable graph
+    let mut graph = Graph::from_snapshot(snapshot);
+
+    // multiple packages are resolved in alphabetical order
+    package_reqs.sort_by(|a, b| a.name.cmp(&b.name));
+
+    // go over the top level packages first, then down the
+    // tree one level at a time through all the branches
+    let mut unresolved_tasks = Vec::with_capacity(package_reqs.len());
+    for package_req in package_reqs {
+      if graph.has_package_req(&package_req) {
+        // skip analyzing this package, as there's already a matching top level package
+        continue;
+      }
+
+      // no existing best version, so resolve the current packages
+      let api = self.api.clone();
+      let maybe_info = if should_sync_download() {
+        // for deterministic test output
+        Some(api.package_info(&package_req.name).await)
+      } else {
+        None
+      };
+      unresolved_tasks.push(tokio::task::spawn(async move {
+        let info = match maybe_info {
+          Some(info) => info?,
+          None => api.package_info(&package_req.name).await?,
+        };
+        Result::<_, AnyError>::Ok((package_req, info))
+      }));
+    }
+
+    let mut resolver = GraphDependencyResolver::new(&mut graph, &self.api);
+
+    for result in futures::future::join_all(unresolved_tasks).await {
+      let (package_req, info) = result??;
+      resolver.add_package_req(&package_req, info)?;
+    }
+
+    resolver.resolve_pending().await?;
+
+    graph.into_snapshot(&self.api).await
+  }
+
+  pub fn resolve_package_from_id(
+    &self,
+    id: &NpmPackageId,
+  ) -> Option<NpmResolutionPackage> {
+    self.snapshot.read().package_from_id(id).cloned()
+  }
+
+  pub fn resolve_package_cache_folder_id_from_id(
+    &self,
+    id: &NpmPackageId,
+  ) -> Option<NpmPackageCacheFolderId> {
+    self
+      .snapshot
+      .read()
+      .package_from_id(id)
+      .map(|p| p.get_package_cache_folder_id())
+  }
+
+  pub fn resolve_package_from_package(
+    &self,
+    name: &str,
+    referrer: &NpmPackageCacheFolderId,
+  ) -> Result<NpmResolutionPackage, AnyError> {
+    self
+      .snapshot
+      .read()
+      .resolve_package_from_package(name, referrer)
+      .cloned()
+  }
+
+  /// Resolve a node package from a deno module.
+  pub fn resolve_package_from_deno_module(
+    &self,
+    package: &NpmPackageReq,
+  ) -> Result<NpmResolutionPackage, AnyError> {
+    self
+      .snapshot
+      .read()
+      .resolve_package_from_deno_module(package)
+      .cloned()
+  }
+
+  pub fn all_packages(&self) -> Vec<NpmResolutionPackage> {
+    self.snapshot.read().all_packages()
+  }
+
+  pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned {
+    self.snapshot.read().all_packages_partitioned()
+  }
+
+  pub fn has_packages(&self) -> bool {
+    !self.snapshot.read().packages.is_empty()
+  }
+
+  pub fn snapshot(&self) -> NpmResolutionSnapshot {
+    self.snapshot.read().clone()
+  }
+
+  pub fn lock(
+    &self,
+    lockfile: &mut Lockfile,
+    snapshot: &NpmResolutionSnapshot,
+  ) -> Result<(), AnyError> {
+    for (package_req, package_id) in snapshot.package_reqs.iter() {
+      lockfile.insert_npm_specifier(package_req, package_id);
+    }
+    for package in self.all_packages() {
+      lockfile.check_or_insert_npm_package(&package)?;
+    }
+    Ok(())
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn parse_npm_package_ref() {
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package/test").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "@package/test".to_string(),
+          version_req: None,
+        },
+        sub_path: None,
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package/test@1").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "@package/test".to_string(),
+          version_req: Some(SpecifierVersionReq::parse("1").unwrap()),
+        },
+        sub_path: None,
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package/test@~1.1/sub_path").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "@package/test".to_string(),
+          version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()),
+        },
+        sub_path: Some("sub_path".to_string()),
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "@package/test".to_string(),
+          version_req: None,
+        },
+        sub_path: Some("sub_path".to_string()),
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:test").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "test".to_string(),
+          version_req: None,
+        },
+        sub_path: None,
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:test@^1.2").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "test".to_string(),
+          version_req: Some(SpecifierVersionReq::parse("^1.2").unwrap()),
+        },
+        sub_path: None,
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:test@~1.1/sub_path").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "test".to_string(),
+          version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()),
+        },
+        sub_path: Some("sub_path".to_string()),
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(),
+      NpmPackageReference {
+        req: NpmPackageReq {
+          name: "@package/test".to_string(),
+          version_req: None,
+        },
+        sub_path: Some("sub_path".to_string()),
+      }
+    );
+
+    assert_eq!(
+      NpmPackageReference::from_str("npm:@package")
+        .err()
+        .unwrap()
+        .to_string(),
+      "Not a valid package: @package"
+    );
+  }
+
+  #[test]
+  fn serialize_npm_package_id() {
+    let id = NpmPackageId {
+      name: "pkg-a".to_string(),
+      version: NpmVersion::parse("1.2.3").unwrap(),
+      peer_dependencies: vec![
+        NpmPackageId {
+          name: "pkg-b".to_string(),
+          version: NpmVersion::parse("3.2.1").unwrap(),
+          peer_dependencies: vec![
+            NpmPackageId {
+              name: "pkg-c".to_string(),
+              version: NpmVersion::parse("1.3.2").unwrap(),
+              peer_dependencies: vec![],
+            },
+            NpmPackageId {
+              name: "pkg-d".to_string(),
+              version: NpmVersion::parse("2.3.4").unwrap(),
+              peer_dependencies: vec![],
+            },
+          ],
+        },
+        NpmPackageId {
+          name: "pkg-e".to_string(),
+          version: NpmVersion::parse("2.3.1").unwrap(),
+          peer_dependencies: vec![NpmPackageId {
+            name: "pkg-f".to_string(),
+            version: NpmVersion::parse("2.3.1").unwrap(),
+            peer_dependencies: vec![],
+          }],
+        },
+      ],
+    };
+    let serialized = id.as_serialized();
+    assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1");
+    assert_eq!(NpmPackageId::from_serialized(&serialized).unwrap(), id);
+  }
+}

cli/npm/resolution/snapshot.rs

@@ -0,0 +1,470 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use deno_core::anyhow::anyhow;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::futures;
+use deno_core::parking_lot::Mutex;
+use serde::Deserialize;
+use serde::Serialize;
+
+use crate::lockfile::Lockfile;
+use crate::npm::cache::should_sync_download;
+use crate::npm::cache::NpmPackageCacheFolderId;
+use crate::npm::registry::NpmPackageVersionDistInfo;
+use crate::npm::registry::NpmRegistryApi;
+use crate::npm::registry::RealNpmRegistryApi;
+
+use super::NpmPackageId;
+use super::NpmPackageReq;
+use super::NpmResolutionPackage;
+use super::NpmVersionMatcher;
+
+/// Packages partitioned by if they are "copy" packages or not.
+pub struct NpmPackagesPartitioned {
+  pub packages: Vec<NpmResolutionPackage>,
+  /// Since peer dependency resolution occurs based on ancestors and ancestor
+  /// siblings, this may sometimes cause the same package (name and version)
+  /// to have different dependencies based on where it appears in the tree.
+  /// For these packages, we create a "copy package" or duplicate of the package
+  /// whose dependencies are that of where in the tree they've resolved to.
+  pub copy_packages: Vec<NpmResolutionPackage>,
+}
+
+impl NpmPackagesPartitioned {
+  pub fn into_all(self) -> Vec<NpmResolutionPackage> {
+    let mut packages = self.packages;
+    packages.extend(self.copy_packages);
+    packages
+  }
+}
+
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+pub struct NpmResolutionSnapshot {
+  #[serde(with = "map_to_vec")]
+  pub(super) package_reqs: HashMap<NpmPackageReq, NpmPackageId>,
+  pub(super) packages_by_name: HashMap<String, Vec<NpmPackageId>>,
+  #[serde(with = "map_to_vec")]
+  pub(super) packages: HashMap<NpmPackageId, NpmResolutionPackage>,
+}
+
+// This is done so the maps with non-string keys get serialized and deserialized as vectors.
+// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792
+mod map_to_vec {
+  use std::collections::HashMap;
+
+  use serde::de::Deserialize;
+  use serde::de::Deserializer;
+  use serde::ser::Serializer;
+  use serde::Serialize;
+
+  pub fn serialize<S, K: Serialize, V: Serialize>(
+    map: &HashMap<K, V>,
+    serializer: S,
+  ) -> Result<S::Ok, S::Error>
+  where
+    S: Serializer,
+  {
+    serializer.collect_seq(map.iter())
+  }
+
+  pub fn deserialize<
+    'de,
+    D,
+    K: Deserialize<'de> + Eq + std::hash::Hash,
+    V: Deserialize<'de>,
+  >(
+    deserializer: D,
+  ) -> Result<HashMap<K, V>, D::Error>
+  where
+    D: Deserializer<'de>,
+  {
+    let mut map = HashMap::new();
+    for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? {
+      map.insert(key, value);
+    }
+    Ok(map)
+  }
+}
+
+impl NpmResolutionSnapshot {
+  /// Resolve a node package from a deno module.
+  pub fn resolve_package_from_deno_module(
+    &self,
+    req: &NpmPackageReq,
+  ) -> Result<&NpmResolutionPackage, AnyError> {
+    match self.package_reqs.get(req) {
+      Some(id) => Ok(self.packages.get(id).unwrap()),
+      None => bail!("could not find npm package directory for '{}'", req),
+    }
+  }
+
+  pub fn top_level_packages(&self) -> Vec<NpmPackageId> {
+    self
+      .package_reqs
+      .values()
+      .cloned()
+      .collect::<HashSet<_>>()
+      .into_iter()
+      .collect::<Vec<_>>()
+  }
+
+  pub fn package_from_id(
+    &self,
+    id: &NpmPackageId,
+  ) -> Option<&NpmResolutionPackage> {
+    self.packages.get(id)
+  }
+
+  pub fn resolve_package_from_package(
+    &self,
+    name: &str,
+    referrer: &NpmPackageCacheFolderId,
+  ) -> Result<&NpmResolutionPackage, AnyError> {
+    // todo(dsherret): do we need an additional hashmap to get this quickly?
+    let referrer_package = self
+      .packages_by_name
+      .get(&referrer.name)
+      .and_then(|packages| {
+        packages
+          .iter()
+          .filter(|p| p.version == referrer.version)
+          .filter_map(|id| {
+            let package = self.packages.get(id)?;
+            if package.copy_index == referrer.copy_index {
+              Some(package)
+            } else {
+              None
+            }
+          })
+          .next()
+      })
+      .ok_or_else(|| {
+        anyhow!("could not find referrer npm package '{}'", referrer)
+      })?;
+
+    let name = name_without_path(name);
+    if let Some(id) = referrer_package.dependencies.get(name) {
+      return Ok(self.packages.get(id).unwrap());
+    }
+
+    if referrer_package.id.name == name {
+      return Ok(referrer_package);
+    }
+
+    // TODO(bartlomieju): this should use a reverse lookup table in the
+    // snapshot instead of resolving best version again.
+    let req = NpmPackageReq {
+      name: name.to_string(),
+      version_req: None,
+    };
+
+    if let Some(id) = self.resolve_best_package_id(name, &req) {
+      if let Some(pkg) = self.packages.get(&id) {
+        return Ok(pkg);
+      }
+    }
+
+    bail!(
+      "could not find npm package '{}' referenced by '{}'",
+      name,
+      referrer
+    )
+  }
+
+  pub fn all_packages(&self) -> Vec<NpmResolutionPackage> {
+    self.packages.values().cloned().collect()
+  }
+
+  pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned {
+    let mut packages = self.all_packages();
+    let mut copy_packages = Vec::with_capacity(packages.len() / 2); // at most 1 copy for every package
+
+    // partition out any packages that are "copy" packages
+    for i in (0..packages.len()).rev() {
+      if packages[i].copy_index > 0 {
+        copy_packages.push(packages.swap_remove(i));
+      }
+    }
+
+    NpmPackagesPartitioned {
+      packages,
+      copy_packages,
+    }
+  }
+
+  pub fn resolve_best_package_id(
+    &self,
+    name: &str,
+    version_matcher: &impl NpmVersionMatcher,
+  ) -> Option<NpmPackageId> {
+    // todo(dsherret): this is not exactly correct because some ids
+    // will be better than others due to peer dependencies
+    let mut maybe_best_id: Option<&NpmPackageId> = None;
+    if let Some(ids) = self.packages_by_name.get(name) {
+      for id in ids {
+        if version_matcher.matches(&id.version) {
+          let is_best_version = maybe_best_id
+            .as_ref()
+            .map(|best_id| best_id.version.cmp(&id.version).is_lt())
+            .unwrap_or(true);
+          if is_best_version {
+            maybe_best_id = Some(id);
+          }
+        }
+      }
+    }
+    maybe_best_id.cloned()
+  }
+
+  pub async fn from_lockfile(
+    lockfile: Arc<Mutex<Lockfile>>,
+    api: &RealNpmRegistryApi,
+  ) -> Result<Self, AnyError> {
+    let mut package_reqs: HashMap<NpmPackageReq, NpmPackageId>;
+    let mut packages_by_name: HashMap<String, Vec<NpmPackageId>>;
+    let mut packages: HashMap<NpmPackageId, NpmResolutionPackage>;
+    let mut copy_index_resolver: SnapshotPackageCopyIndexResolver;
+
+    {
+      let lockfile = lockfile.lock();
+
+      // pre-allocate collections
+      package_reqs =
+        HashMap::with_capacity(lockfile.content.npm.specifiers.len());
+      let packages_len = lockfile.content.npm.packages.len();
+      packages = HashMap::with_capacity(packages_len);
+      packages_by_name = HashMap::with_capacity(packages_len); // close enough
+      copy_index_resolver =
+        SnapshotPackageCopyIndexResolver::with_capacity(packages_len);
+      let mut verify_ids = HashSet::with_capacity(packages_len);
+
+      // collect the specifiers to version mappings
+      for (key, value) in &lockfile.content.npm.specifiers {
+        let package_req = NpmPackageReq::from_str(key)
+          .with_context(|| format!("Unable to parse npm specifier: {}", key))?;
+        let package_id = NpmPackageId::from_serialized(value)?;
+        package_reqs.insert(package_req, package_id.clone());
+        verify_ids.insert(package_id.clone());
+      }
+
+      // then the packages
+      for (key, value) in &lockfile.content.npm.packages {
+        let package_id = NpmPackageId::from_serialized(key)?;
+
+        // collect the dependencies
+        let mut dependencies = HashMap::default();
+
+        packages_by_name
+          .entry(package_id.name.to_string())
+          .or_default()
+          .push(package_id.clone());
+
+        for (name, specifier) in &value.dependencies {
+          let dep_id = NpmPackageId::from_serialized(specifier)?;
+          dependencies.insert(name.to_string(), dep_id.clone());
+          verify_ids.insert(dep_id);
+        }
+
+        let package = NpmResolutionPackage {
+          id: package_id.clone(),
+          copy_index: copy_index_resolver.resolve(&package_id),
+          // temporary dummy value
+          dist: NpmPackageVersionDistInfo {
+            tarball: "foobar".to_string(),
+            shasum: "foobar".to_string(),
+            integrity: Some("foobar".to_string()),
+          },
+          dependencies,
+        };
+
+        packages.insert(package_id, package);
+      }
+
+      // verify that all these ids exist in packages
+      for id in &verify_ids {
+        if !packages.contains_key(id) {
+          bail!(
+            "the lockfile is corrupt. You can recreate it with --lock-write"
+          );
+        }
+      }
+    }
+
+    let mut unresolved_tasks = Vec::with_capacity(packages_by_name.len());
+
+    // cache the package names in parallel in the registry api
+    // unless synchronous download should occur
+    if should_sync_download() {
+      let mut package_names = packages_by_name.keys().collect::<Vec<_>>();
+      package_names.sort();
+      for package_name in package_names {
+        api.package_info(package_name).await?;
+      }
+    } else {
+      for package_name in packages_by_name.keys() {
+        let package_name = package_name.clone();
+        let api = api.clone();
+        unresolved_tasks.push(tokio::task::spawn(async move {
+          api.package_info(&package_name).await?;
+          Result::<_, AnyError>::Ok(())
+        }));
+      }
+    }
+    for result in futures::future::join_all(unresolved_tasks).await {
+      result??;
+    }
+
+    // ensure the dist is set for each package
+    for package in packages.values_mut() {
+      // this will read from the memory cache now
+      let version_info = match api
+        .package_version_info(&package.id.name, &package.id.version)
+        .await?
+      {
+        Some(version_info) => version_info,
+        None => {
+          bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id.display());
+        }
+      };
+      package.dist = version_info.dist;
+    }
+
+    Ok(Self {
+      package_reqs,
+      packages_by_name,
+      packages,
+    })
+  }
+}
+
+pub struct SnapshotPackageCopyIndexResolver {
+  packages_to_copy_index: HashMap<NpmPackageId, usize>,
+  package_name_version_to_copy_count: HashMap<(String, String), usize>,
+}
+
+impl SnapshotPackageCopyIndexResolver {
+  pub fn with_capacity(capacity: usize) -> Self {
+    Self {
+      packages_to_copy_index: HashMap::with_capacity(capacity),
+      package_name_version_to_copy_count: HashMap::with_capacity(capacity), // close enough
+    }
+  }
+
+  pub fn from_map_with_capacity(
+    mut packages_to_copy_index: HashMap<NpmPackageId, usize>,
+    capacity: usize,
+  ) -> Self {
+    let mut package_name_version_to_copy_count =
+      HashMap::with_capacity(capacity); // close enough
+    if capacity > packages_to_copy_index.len() {
+      packages_to_copy_index.reserve(capacity - packages_to_copy_index.len());
+    }
+
+    for (id, index) in &packages_to_copy_index {
+      let entry = package_name_version_to_copy_count
+        .entry((id.name.to_string(), id.version.to_string()))
+        .or_insert(0);
+      if *entry < *index {
+        *entry = *index;
+      }
+    }
+    Self {
+      packages_to_copy_index,
+      package_name_version_to_copy_count,
+    }
+  }
+
+  pub fn resolve(&mut self, id: &NpmPackageId) -> usize {
+    if let Some(index) = self.packages_to_copy_index.get(id) {
+      *index
+    } else {
+      let index = *self
+        .package_name_version_to_copy_count
+        .entry((id.name.to_string(), id.version.to_string()))
+        .and_modify(|count| {
+          *count += 1;
+        })
+        .or_insert(0);
+      self.packages_to_copy_index.insert(id.clone(), index);
+      index
+    }
+  }
+}
+
+fn name_without_path(name: &str) -> &str {
+  let mut search_start_index = 0;
+  if name.starts_with('@') {
+    if let Some(slash_index) = name.find('/') {
+      search_start_index = slash_index + 1;
+    }
+  }
+  if let Some(slash_index) = &name[search_start_index..].find('/') {
+    // get the name up until the path slash
+    &name[0..search_start_index + slash_index]
+  } else {
+    name
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_name_without_path() {
+    assert_eq!(name_without_path("foo"), "foo");
+    assert_eq!(name_without_path("@foo/bar"), "@foo/bar");
+    assert_eq!(name_without_path("@foo/bar/baz"), "@foo/bar");
+    assert_eq!(name_without_path("@hello"), "@hello");
+  }
+
+  #[test]
+  fn test_copy_index_resolver() {
+    let mut copy_index_resolver =
+      SnapshotPackageCopyIndexResolver::with_capacity(10);
+    assert_eq!(
+      copy_index_resolver
+        .resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()),
+      0
+    );
+    assert_eq!(
+      copy_index_resolver
+        .resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()),
+      0
+    );
+    assert_eq!(
+      copy_index_resolver.resolve(
+        &NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0")
+          .unwrap()
+      ),
+      1
+    );
+    assert_eq!(
+      copy_index_resolver.resolve(
+        &NpmPackageId::from_serialized(
+          "package@1.0.0_package-b@1.0.0__package-c@2.0.0"
+        )
+        .unwrap()
+      ),
+      2
+    );
+    assert_eq!(
+      copy_index_resolver.resolve(
+        &NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0")
+          .unwrap()
+      ),
+      1
+    );
+    assert_eq!(
+      copy_index_resolver
+        .resolve(&NpmPackageId::from_serialized("package-b@1.0.0").unwrap()),
+      0
+    );
+  }
+}