deno/cli/npm/managed.rs
Nathan Whitaker 780b741555
perf(npm): load npm resolution snapshot directly from lockfile (#28647)
Fixes #27264. Fixes https://github.com/denoland/deno/issues/28161.

Currently the new lockfile version is gated behind an unstable flag
(`--unstable-lockfile-v5`) until the next minor release, where it will
become the default.

The main motivation here is that it improves startup performance when
using the global cache or `--node-modules-dir=auto`.

In a create-next-app project, running an empty file:
```
❯ hyperfine --warmup 25 -N --setup "rm -f deno.lock" "deno run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js" "deno run --node-modules-dir=manual -A empty.js" "deno-this-pr run --node-modules-dir=manual -A empty.js"
Benchmark 1: deno run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     247.6 ms ±   1.7 ms    [User: 228.7 ms, System: 19.0 ms]
  Range (min … max):   245.5 ms … 251.5 ms    12 runs

Benchmark 2: deno-this-pr run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     169.8 ms ±   1.0 ms    [User: 152.9 ms, System: 17.9 ms]
  Range (min … max):   168.9 ms … 172.5 ms    17 runs

Benchmark 3: deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
  Time (mean ± σ):      16.2 ms ±   0.7 ms    [User: 12.3 ms, System: 5.7 ms]
  Range (min … max):    15.2 ms …  19.2 ms    185 runs

Benchmark 4: deno run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.2 ms ±   0.8 ms    [User: 11.6 ms, System: 5.5 ms]
  Range (min … max):    14.9 ms …  19.7 ms    187 runs

Benchmark 5: deno-this-pr run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.0 ms ±   0.9 ms    [User: 12.0 ms, System: 5.5 ms]
  Range (min … max):    14.8 ms …  22.3 ms    190 runs

  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.

Summary
  deno-this-pr run --node-modules-dir=manual -A empty.js ran
    1.01 ± 0.08 times faster than deno run --node-modules-dir=manual -A empty.js
    1.01 ± 0.07 times faster than deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
   10.64 ± 0.60 times faster than deno-this-pr run --node-modules-dir=auto -A empty.js
   15.51 ± 0.88 times faster than deno run --node-modules-dir=auto -A empty.js
```

When using the new lockfile version, this leads to a 15.5x faster
startup time compared to the current deno version.

Install times benefit as well, though to a lesser degree.

`deno install` on a create-next-app project, with everything cached
(just setting up node_modules from scratch):

```
❯ hyperfine --warmup 5 -N --prepare "rm -rf node_modules" --setup "rm -rf deno.lock" "deno i" "deno-this-pr i" "deno-this-pr i --unstable-lockfile-v5"
Benchmark 1: deno i
  Time (mean ± σ):     464.4 ms ±   8.8 ms    [User: 227.7 ms, System: 217.3 ms]
  Range (min … max):   452.6 ms … 478.3 ms    10 runs

Benchmark 2: deno-this-pr i
  Time (mean ± σ):     368.8 ms ±  22.0 ms    [User: 150.8 ms, System: 198.1 ms]
  Range (min … max):   344.8 ms … 397.6 ms    10 runs

Benchmark 3: deno-this-pr i --unstable-lockfile-v5
  Time (mean ± σ):     211.9 ms ±  17.1 ms    [User: 7.1 ms, System: 177.2 ms]
  Range (min … max):   191.3 ms … 233.4 ms    10 runs

Summary
  deno-this-pr i --unstable-lockfile-v5 ran
    1.74 ± 0.17 times faster than deno-this-pr i
    2.19 ± 0.18 times faster than deno i
```

With lockfile v5, a 2.19x faster install time compared to the current
deno.
2025-04-08 02:06:17 +00:00

249 lines
7.1 KiB
Rust

// Copyright 2018-2025 the Deno authors. MIT license.
use std::path::PathBuf;
use std::sync::Arc;
use deno_core::parking_lot::Mutex;
use deno_error::JsError;
use deno_error::JsErrorBox;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions;
use deno_resolver::npm::managed::NpmResolutionCell;
use thiserror::Error;
use super::CliNpmRegistryInfoProvider;
use super::WorkspaceNpmPatchPackages;
use crate::args::CliLockfile;
use crate::sys::CliSys;
pub type CliManagedNpmResolverCreateOptions =
ManagedNpmResolverCreateOptions<CliSys>;
#[derive(Debug, Clone)]
pub enum CliNpmResolverManagedSnapshotOption {
ResolveFromLockfile(Arc<CliLockfile>),
Specified(Option<ValidSerializedNpmResolutionSnapshot>),
}
#[derive(Debug)]
enum SyncState {
Pending(Option<CliNpmResolverManagedSnapshotOption>),
Err(ResolveSnapshotError),
Success,
}
#[derive(Debug)]
pub struct NpmResolutionInitializer {
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_resolution: Arc<NpmResolutionCell>,
patch_packages: Arc<WorkspaceNpmPatchPackages>,
queue: tokio::sync::Mutex<()>,
sync_state: Mutex<SyncState>,
}
impl NpmResolutionInitializer {
pub fn new(
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_resolution: Arc<NpmResolutionCell>,
patch_packages: Arc<WorkspaceNpmPatchPackages>,
snapshot_option: CliNpmResolverManagedSnapshotOption,
) -> Self {
Self {
npm_registry_info_provider,
npm_resolution,
patch_packages,
queue: tokio::sync::Mutex::new(()),
sync_state: Mutex::new(SyncState::Pending(Some(snapshot_option))),
}
}
#[cfg(debug_assertions)]
pub fn debug_assert_initialized(&self) {
if !matches!(*self.sync_state.lock(), SyncState::Success) {
panic!("debug assert: npm resolution must be initialized before calling this code");
}
}
pub async fn ensure_initialized(&self) -> Result<(), JsErrorBox> {
// fast exit if not pending
{
match &*self.sync_state.lock() {
SyncState::Pending(_) => {}
SyncState::Err(err) => return Err(JsErrorBox::from_err(err.clone())),
SyncState::Success => return Ok(()),
}
}
// only allow one task in here at a time
let _guard = self.queue.lock().await;
let snapshot_option = {
let mut sync_state = self.sync_state.lock();
match &mut *sync_state {
SyncState::Pending(snapshot_option) => {
// this should never panic, but if it does it means that a
// previous future was dropped while initialization occurred...
// that should never happen because this is initialized during
// startup
snapshot_option.take().unwrap()
}
// another thread updated the state while we were waiting
SyncState::Err(resolve_snapshot_error) => {
return Err(JsErrorBox::from_err(resolve_snapshot_error.clone()));
}
SyncState::Success => {
return Ok(());
}
}
};
match resolve_snapshot(
&self.npm_registry_info_provider,
snapshot_option,
&self.patch_packages,
)
.await
{
Ok(maybe_snapshot) => {
if let Some(snapshot) = maybe_snapshot {
self
.npm_resolution
.set_snapshot(NpmResolutionSnapshot::new(snapshot));
}
let mut sync_state = self.sync_state.lock();
*sync_state = SyncState::Success;
Ok(())
}
Err(err) => {
let mut sync_state = self.sync_state.lock();
*sync_state = SyncState::Err(err.clone());
Err(JsErrorBox::from_err(err))
}
}
}
}
#[derive(Debug, Error, Clone, JsError)]
#[error("failed reading lockfile '{}'", lockfile_path.display())]
#[class(inherit)]
pub struct ResolveSnapshotError {
lockfile_path: PathBuf,
#[inherit]
#[source]
source: SnapshotFromLockfileError,
}
impl ResolveSnapshotError {
pub fn maybe_integrity_check_error(
&self,
) -> Option<&deno_npm::resolution::IntegrityCheckFailedError> {
match &self.source {
SnapshotFromLockfileError::SnapshotFromLockfile(
deno_npm::resolution::SnapshotFromLockfileError::IntegrityCheckFailed(
err,
),
) => Some(err),
_ => None,
}
}
}
async fn resolve_snapshot(
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
snapshot: CliNpmResolverManagedSnapshotOption,
patch_packages: &WorkspaceNpmPatchPackages,
) -> Result<Option<ValidSerializedNpmResolutionSnapshot>, ResolveSnapshotError>
{
match snapshot {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(lockfile) => {
if !lockfile.overwrite() {
let snapshot = snapshot_from_lockfile(
lockfile.clone(),
&registry_info_provider.as_npm_registry_api(),
patch_packages,
)
.await
.map_err(|source| ResolveSnapshotError {
lockfile_path: lockfile.filename.clone(),
source,
})?;
Ok(Some(snapshot))
} else {
Ok(None)
}
}
CliNpmResolverManagedSnapshotOption::Specified(snapshot) => Ok(snapshot),
}
}
#[derive(Debug, Error, Clone, JsError)]
pub enum SnapshotFromLockfileError {
// TODO(nathanwhit): remove once we've migrated to lockfile v5
#[error(transparent)]
#[class(inherit)]
IncompleteError(
#[from] deno_npm::resolution::IncompleteSnapshotFromLockfileError,
),
#[error(transparent)]
#[class(inherit)]
SnapshotFromLockfile(#[from] deno_npm::resolution::SnapshotFromLockfileError),
}
pub(crate) struct DefaultTarballUrl;
impl deno_npm::resolution::DefaultTarballUrlProvider for DefaultTarballUrl {
fn default_tarball_url(&self, id: &deno_npm::NpmPackageId) -> String {
let scope = id.nv.scope();
let package_name = if let Some(scope) = scope {
id.nv
.name
.strip_prefix(scope)
.unwrap_or(&id.nv.name)
.trim_start_matches('/')
} else {
&id.nv.name
};
format!(
"https://registry.npmjs.org/{}/-/{}-{}.tgz",
id.nv.name, package_name, id.nv.version
)
}
}
async fn snapshot_from_lockfile(
lockfile: Arc<CliLockfile>,
api: &dyn NpmRegistryApi,
patch_packages: &WorkspaceNpmPatchPackages,
) -> Result<ValidSerializedNpmResolutionSnapshot, SnapshotFromLockfileError> {
if lockfile.use_lockfile_v5() {
let snapshot = deno_npm::resolution::snapshot_from_lockfile_v5(
deno_npm::resolution::SnapshotFromLockfileV5Params {
patch_packages: &patch_packages.0,
lockfile: &lockfile.lock(),
default_tarball_url: &DefaultTarballUrl,
},
)?;
Ok(snapshot)
} else {
let (incomplete_snapshot, skip_integrity_check) = {
let lock = lockfile.lock();
(
deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?,
lock.overwrite,
)
};
let snapshot = deno_npm::resolution::snapshot_from_lockfile(
deno_npm::resolution::SnapshotFromLockfileParams {
incomplete_snapshot,
api,
patch_packages: &patch_packages.0,
skip_integrity_check,
},
)
.await?;
Ok(snapshot)
}
}