deno/cli/tools/pm/cache_deps.rs
Nathan Whitaker 780b741555
perf(npm): load npm resolution snapshot directly from lockfile (#28647)
Fixes #27264. Fixes https://github.com/denoland/deno/issues/28161.

Currently the new lockfile version is gated behind an unstable flag
(`--unstable-lockfile-v5`) until the next minor release, where it will
become the default.

The main motivation here is that it improves startup performance when
using the global cache or `--node-modules-dir=auto`.

In a create-next-app project, running an empty file:
```
❯ hyperfine --warmup 25 -N --setup "rm -f deno.lock" "deno run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js" "deno run --node-modules-dir=manual -A empty.js" "deno-this-pr run --node-modules-dir=manual -A empty.js"
Benchmark 1: deno run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     247.6 ms ±   1.7 ms    [User: 228.7 ms, System: 19.0 ms]
  Range (min … max):   245.5 ms … 251.5 ms    12 runs

Benchmark 2: deno-this-pr run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     169.8 ms ±   1.0 ms    [User: 152.9 ms, System: 17.9 ms]
  Range (min … max):   168.9 ms … 172.5 ms    17 runs

Benchmark 3: deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
  Time (mean ± σ):      16.2 ms ±   0.7 ms    [User: 12.3 ms, System: 5.7 ms]
  Range (min … max):    15.2 ms …  19.2 ms    185 runs

Benchmark 4: deno run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.2 ms ±   0.8 ms    [User: 11.6 ms, System: 5.5 ms]
  Range (min … max):    14.9 ms …  19.7 ms    187 runs

Benchmark 5: deno-this-pr run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.0 ms ±   0.9 ms    [User: 12.0 ms, System: 5.5 ms]
  Range (min … max):    14.8 ms …  22.3 ms    190 runs

  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.

Summary
  deno-this-pr run --node-modules-dir=manual -A empty.js ran
    1.01 ± 0.08 times faster than deno run --node-modules-dir=manual -A empty.js
    1.01 ± 0.07 times faster than deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
   10.64 ± 0.60 times faster than deno-this-pr run --node-modules-dir=auto -A empty.js
   15.51 ± 0.88 times faster than deno run --node-modules-dir=auto -A empty.js
```

When using the new lockfile version, this leads to a 15.5x faster
startup time compared to the current deno version.

Install times benefit as well, though to a lesser degree.

`deno install` on a create-next-app project, with everything cached
(just setting up node_modules from scratch):

```
❯ hyperfine --warmup 5 -N --prepare "rm -rf node_modules" --setup "rm -rf deno.lock" "deno i" "deno-this-pr i" "deno-this-pr i --unstable-lockfile-v5"
Benchmark 1: deno i
  Time (mean ± σ):     464.4 ms ±   8.8 ms    [User: 227.7 ms, System: 217.3 ms]
  Range (min … max):   452.6 ms … 478.3 ms    10 runs

Benchmark 2: deno-this-pr i
  Time (mean ± σ):     368.8 ms ±  22.0 ms    [User: 150.8 ms, System: 198.1 ms]
  Range (min … max):   344.8 ms … 397.6 ms    10 runs

Benchmark 3: deno-this-pr i --unstable-lockfile-v5
  Time (mean ± σ):     211.9 ms ±  17.1 ms    [User: 7.1 ms, System: 177.2 ms]
  Range (min … max):   191.3 ms … 233.4 ms    10 runs

Summary
  deno-this-pr i --unstable-lockfile-v5 ran
    1.74 ± 0.17 times faster than deno-this-pr i
    2.19 ± 0.18 times faster than deno i
```

With lockfile v5, a 2.19x faster install time compared to the current
deno.
2025-04-08 02:06:17 +00:00

180 lines
5.4 KiB
Rust

// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::HashSet;
use std::sync::Arc;
use deno_core::error::AnyError;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::StreamExt;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::Version;
use crate::factory::CliFactory;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::npm::installer::PackageCaching;
pub async fn cache_top_level_deps(
// todo(dsherret): don't pass the factory into this function. Instead use ctor deps
factory: &CliFactory,
jsr_resolver: Option<Arc<crate::jsr::JsrFetchResolver>>,
) -> Result<(), AnyError> {
let npm_installer = factory.npm_installer().await?;
npm_installer
.ensure_top_level_package_json_install()
.await?;
if let Some(lockfile) = factory.maybe_lockfile().await? {
lockfile.error_if_changed()?;
}
// cache as many entries in the import map as we can
let resolver = factory.workspace_resolver().await?;
let mut maybe_graph_error = Ok(());
if let Some(import_map) = resolver.maybe_import_map() {
let jsr_resolver = if let Some(resolver) = jsr_resolver {
resolver
} else {
Arc::new(crate::jsr::JsrFetchResolver::new(
factory.file_fetcher()?.clone(),
))
};
let mut graph_permit = factory
.main_module_graph_container()
.await?
.acquire_update_permit()
.await;
let graph = graph_permit.graph_mut();
if let Some(lockfile) = factory.maybe_lockfile().await? {
let lockfile = lockfile.lock();
crate::graph_util::fill_graph_from_lockfile(graph, &lockfile);
}
let mut roots = Vec::new();
let mut info_futures = FuturesUnordered::new();
let mut seen_reqs = HashSet::new();
let workspace_npm_packages = resolver
.package_jsons()
.filter_map(|pkg_json| {
pkg_json
.name
.as_deref()
.and_then(|name| Some((name, pkg_json.version.as_deref()?)))
})
.collect::<HashMap<_, _>>();
for entry in import_map.imports().entries().chain(
import_map
.scopes()
.flat_map(|scope| scope.imports.entries()),
) {
let Some(specifier) = entry.value else {
continue;
};
match specifier.scheme() {
"jsr" => {
let specifier_str = specifier.as_str();
if let Ok(req) = JsrPackageReqReference::from_str(specifier_str) {
if let Some(sub_path) = req.sub_path() {
if sub_path.ends_with('/') {
continue;
}
roots.push(specifier.clone());
continue;
}
if !seen_reqs.insert(req.req().clone()) {
continue;
}
let resolved_req = graph.packages.mappings().get(req.req());
let jsr_resolver = jsr_resolver.clone();
info_futures.push(async move {
let nv = if let Some(req) = resolved_req {
Cow::Borrowed(req)
} else {
Cow::Owned(jsr_resolver.req_to_nv(req.req()).await?)
};
if let Some(info) = jsr_resolver.package_version_info(&nv).await {
return Some((specifier.clone(), info));
}
None
});
}
}
"npm" => {
let Ok(req_ref) =
NpmPackageReqReference::from_str(specifier.as_str())
else {
continue;
};
let version = workspace_npm_packages.get(&*req_ref.req().name);
if let Some(version) = version {
let Ok(version) = Version::parse_from_npm(version) else {
continue;
};
let version_req = &req_ref.req().version_req;
if version_req.tag().is_none() && version_req.matches(&version) {
// if version req matches the workspace package's version, use that
// (so it doesn't need to be installed)
continue;
}
}
roots.push(specifier.clone())
}
_ => {
if entry.key.ends_with('/') && specifier.as_str().ends_with('/') {
continue;
}
if specifier.scheme() == "file" {
if let Ok(path) = specifier.to_file_path() {
if !path.is_file() {
continue;
}
}
}
roots.push(specifier.clone());
}
}
}
while let Some(info_future) = info_futures.next().await {
if let Some((specifier, info)) = info_future {
let exports = info.exports();
for (k, _) in exports {
if let Ok(spec) = specifier.join(k) {
roots.push(spec);
}
}
}
}
drop(info_futures);
let graph_builder = factory.module_graph_builder().await?;
graph_builder
.build_graph_with_npm_resolution(
graph,
CreateGraphOptions {
loader: None,
graph_kind: graph.graph_kind(),
is_dynamic: false,
roots: roots.clone(),
npm_caching: crate::graph_util::NpmCachingStrategy::Manual,
},
)
.await?;
maybe_graph_error = graph_builder.graph_roots_valid(graph, &roots, true);
}
npm_installer.cache_packages(PackageCaching::All).await?;
maybe_graph_error?;
Ok(())
}