deno/cli/npm/installer/global.rs
Nathan Whitaker 780b741555
perf(npm): load npm resolution snapshot directly from lockfile (#28647)
Fixes #27264. Fixes https://github.com/denoland/deno/issues/28161.

Currently the new lockfile version is gated behind an unstable flag
(`--unstable-lockfile-v5`) until the next minor release, where it will
become the default.

The main motivation here is that it improves startup performance when
using the global cache or `--node-modules-dir=auto`.

In a create-next-app project, running an empty file:
```
❯ hyperfine --warmup 25 -N --setup "rm -f deno.lock" "deno run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js" "deno run --node-modules-dir=manual -A empty.js" "deno-this-pr run --node-modules-dir=manual -A empty.js"
Benchmark 1: deno run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     247.6 ms ±   1.7 ms    [User: 228.7 ms, System: 19.0 ms]
  Range (min … max):   245.5 ms … 251.5 ms    12 runs

Benchmark 2: deno-this-pr run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     169.8 ms ±   1.0 ms    [User: 152.9 ms, System: 17.9 ms]
  Range (min … max):   168.9 ms … 172.5 ms    17 runs

Benchmark 3: deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
  Time (mean ± σ):      16.2 ms ±   0.7 ms    [User: 12.3 ms, System: 5.7 ms]
  Range (min … max):    15.2 ms …  19.2 ms    185 runs

Benchmark 4: deno run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.2 ms ±   0.8 ms    [User: 11.6 ms, System: 5.5 ms]
  Range (min … max):    14.9 ms …  19.7 ms    187 runs

Benchmark 5: deno-this-pr run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.0 ms ±   0.9 ms    [User: 12.0 ms, System: 5.5 ms]
  Range (min … max):    14.8 ms …  22.3 ms    190 runs

  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.

Summary
  deno-this-pr run --node-modules-dir=manual -A empty.js ran
    1.01 ± 0.08 times faster than deno run --node-modules-dir=manual -A empty.js
    1.01 ± 0.07 times faster than deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
   10.64 ± 0.60 times faster than deno-this-pr run --node-modules-dir=auto -A empty.js
   15.51 ± 0.88 times faster than deno run --node-modules-dir=auto -A empty.js
```

When using the new lockfile version, this leads to a 15.5x faster
startup time compared to the current deno version.

Install times benefit as well, though to a lesser degree.

`deno install` on a create-next-app project, with everything cached
(just setting up node_modules from scratch):

```
❯ hyperfine --warmup 5 -N --prepare "rm -rf node_modules" --setup "rm -rf deno.lock" "deno i" "deno-this-pr i" "deno-this-pr i --unstable-lockfile-v5"
Benchmark 1: deno i
  Time (mean ± σ):     464.4 ms ±   8.8 ms    [User: 227.7 ms, System: 217.3 ms]
  Range (min … max):   452.6 ms … 478.3 ms    10 runs

Benchmark 2: deno-this-pr i
  Time (mean ± σ):     368.8 ms ±  22.0 ms    [User: 150.8 ms, System: 198.1 ms]
  Range (min … max):   344.8 ms … 397.6 ms    10 runs

Benchmark 3: deno-this-pr i --unstable-lockfile-v5
  Time (mean ± σ):     211.9 ms ±  17.1 ms    [User: 7.1 ms, System: 177.2 ms]
  Range (min … max):   191.3 ms … 233.4 ms    10 runs

Summary
  deno-this-pr i --unstable-lockfile-v5 ran
    1.74 ± 0.17 times faster than deno-this-pr i
    2.19 ± 0.18 times faster than deno i
```

With lockfile v5, a 2.19x faster install time compared to the current
deno.
2025-04-08 02:06:17 +00:00

212 lines
6.2 KiB
Rust

// Copyright 2018-2025 the Deno authors. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use deno_core::futures::stream::FuturesUnordered;
use deno_core::futures::StreamExt;
use deno_error::JsErrorBox;
use deno_lib::util::hash::FastInsecureHasher;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_resolver::npm::managed::NpmResolutionCell;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
use super::common::NpmPackageFsInstaller;
use super::PackageCaching;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
/// Resolves packages from the global npm cache.
pub struct GlobalNpmPackageInstaller {
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolutionCell>,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
}
impl std::fmt::Debug for GlobalNpmPackageInstaller {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GlobalNpmPackageInstaller")
.field("cache", &self.cache)
.field("tarball_cache", &self.tarball_cache)
.field("resolution", &self.resolution)
.field("lifecycle_scripts", &self.lifecycle_scripts)
.field("system_info", &self.system_info)
.finish()
}
}
impl GlobalNpmPackageInstaller {
pub fn new(
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolutionCell>,
lifecycle_scripts: LifecycleScriptsConfig,
system_info: NpmSystemInfo,
) -> Self {
Self {
cache,
tarball_cache,
resolution,
lifecycle_scripts,
system_info,
}
}
}
#[async_trait(?Send)]
impl NpmPackageFsInstaller for GlobalNpmPackageInstaller {
async fn cache_packages<'a>(
&self,
caching: PackageCaching<'a>,
) -> Result<(), JsErrorBox> {
let package_partitions = match caching {
PackageCaching::All => self
.resolution
.all_system_packages_partitioned(&self.system_info),
PackageCaching::Only(reqs) => self
.resolution
.subset(&reqs)
.all_system_packages_partitioned(&self.system_info),
};
cache_packages(&package_partitions.packages, &self.tarball_cache)
.await
.map_err(JsErrorBox::from_err)?;
// create the copy package folders
for copy in package_partitions.copy_packages {
self
.cache
.ensure_copy_package(&copy.get_package_cache_folder_id())
.map_err(JsErrorBox::from_err)?;
}
let mut lifecycle_scripts =
super::common::lifecycle_scripts::LifecycleScripts::new(
&self.lifecycle_scripts,
GlobalLifecycleScripts::new(self, &self.lifecycle_scripts.root_dir),
);
// For the global cache, we don't run scripts so we just care that there _are_
// scripts. Kind of hacky, but avoids fetching the "extra" info from the registry.
let extra = deno_npm::NpmPackageExtraInfo {
deprecated: None,
bin: None,
scripts: [("postinstall".into(), "".into())].into_iter().collect(),
};
for package in &package_partitions.packages {
if package.has_scripts {
let package_folder = self.cache.package_folder_for_nv(&package.id.nv);
lifecycle_scripts.add(package, &extra, Cow::Borrowed(&package_folder));
}
}
lifecycle_scripts
.warn_not_run_scripts()
.map_err(JsErrorBox::from_err)?;
Ok(())
}
}
async fn cache_packages(
packages: &[NpmResolutionPackage],
tarball_cache: &Arc<CliNpmTarballCache>,
) -> Result<(), deno_npm_cache::EnsurePackageError> {
let mut futures_unordered = FuturesUnordered::new();
for package in packages {
if let Some(dist) = &package.dist {
futures_unordered.push(async move {
tarball_cache.ensure_package(&package.id.nv, dist).await
});
}
}
while let Some(result) = futures_unordered.next().await {
// surface the first error
result?;
}
Ok(())
}
struct GlobalLifecycleScripts<'a> {
installer: &'a GlobalNpmPackageInstaller,
path_hash: u64,
}
impl<'a> GlobalLifecycleScripts<'a> {
fn new(installer: &'a GlobalNpmPackageInstaller, root_dir: &Path) -> Self {
let mut hasher = FastInsecureHasher::new_without_deno_version();
hasher.write(root_dir.to_string_lossy().as_bytes());
let path_hash = hasher.finish();
Self {
installer,
path_hash,
}
}
fn warned_scripts_file(&self, package: &NpmResolutionPackage) -> PathBuf {
self
.package_path(package)
.join(format!(".scripts-warned-{}", self.path_hash))
}
}
impl super::common::lifecycle_scripts::LifecycleScriptsStrategy
for GlobalLifecycleScripts<'_>
{
fn can_run_scripts(&self) -> bool {
false
}
fn package_path(&self, package: &NpmResolutionPackage) -> PathBuf {
self.installer.cache.package_folder_for_nv(&package.id.nv)
}
fn warn_on_scripts_not_run(
&self,
packages: &[(&NpmResolutionPackage, PathBuf)],
) -> std::result::Result<(), std::io::Error> {
log::warn!("{} The following packages contained npm lifecycle scripts ({}) that were not executed:", colors::yellow("Warning"), colors::gray("preinstall/install/postinstall"));
for (package, _) in packages {
log::warn!("┠─ {}", colors::gray(format!("npm:{}", package.id.nv)));
}
log::warn!("");
log::warn!(
"┠─ {}",
colors::italic("This may cause the packages to not work correctly.")
);
log::warn!("┠─ {}", colors::italic("Lifecycle scripts are only supported when using a `node_modules` directory."));
log::warn!(
"┠─ {}",
colors::italic("Enable it in your deno config file:")
);
log::warn!("┖─ {}", colors::bold("\"nodeModulesDir\": \"auto\""));
for (package, _) in packages {
std::fs::write(self.warned_scripts_file(package), "")?;
}
Ok(())
}
fn did_run_scripts(
&self,
_package: &NpmResolutionPackage,
) -> Result<(), std::io::Error> {
Ok(())
}
fn has_warned(&self, package: &NpmResolutionPackage) -> bool {
self.warned_scripts_file(package).exists()
}
fn has_run(&self, _package: &NpmResolutionPackage) -> bool {
false
}
}