mirror of
https://github.com/denoland/deno.git
synced 2025-10-03 07:34:36 +00:00

Fixes #27264. Fixes https://github.com/denoland/deno/issues/28161. Currently the new lockfile version is gated behind an unstable flag (`--unstable-lockfile-v5`) until the next minor release, where it will become the default. The main motivation here is that it improves startup performance when using the global cache or `--node-modules-dir=auto`. In a create-next-app project, running an empty file: ``` ❯ hyperfine --warmup 25 -N --setup "rm -f deno.lock" "deno run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js" "deno run --node-modules-dir=manual -A empty.js" "deno-this-pr run --node-modules-dir=manual -A empty.js" Benchmark 1: deno run --node-modules-dir=auto -A empty.js Time (mean ± σ): 247.6 ms ± 1.7 ms [User: 228.7 ms, System: 19.0 ms] Range (min … max): 245.5 ms … 251.5 ms 12 runs Benchmark 2: deno-this-pr run --node-modules-dir=auto -A empty.js Time (mean ± σ): 169.8 ms ± 1.0 ms [User: 152.9 ms, System: 17.9 ms] Range (min … max): 168.9 ms … 172.5 ms 17 runs Benchmark 3: deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js Time (mean ± σ): 16.2 ms ± 0.7 ms [User: 12.3 ms, System: 5.7 ms] Range (min … max): 15.2 ms … 19.2 ms 185 runs Benchmark 4: deno run --node-modules-dir=manual -A empty.js Time (mean ± σ): 16.2 ms ± 0.8 ms [User: 11.6 ms, System: 5.5 ms] Range (min … max): 14.9 ms … 19.7 ms 187 runs Benchmark 5: deno-this-pr run --node-modules-dir=manual -A empty.js Time (mean ± σ): 16.0 ms ± 0.9 ms [User: 12.0 ms, System: 5.5 ms] Range (min … max): 14.8 ms … 22.3 ms 190 runs Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options. Summary deno-this-pr run --node-modules-dir=manual -A empty.js ran 1.01 ± 0.08 times faster than deno run --node-modules-dir=manual -A empty.js 1.01 ± 0.07 times faster than deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js 10.64 ± 0.60 times faster than deno-this-pr run --node-modules-dir=auto -A empty.js 15.51 ± 0.88 times faster than deno run --node-modules-dir=auto -A empty.js ``` When using the new lockfile version, this leads to a 15.5x faster startup time compared to the current deno version. Install times benefit as well, though to a lesser degree. `deno install` on a create-next-app project, with everything cached (just setting up node_modules from scratch): ``` ❯ hyperfine --warmup 5 -N --prepare "rm -rf node_modules" --setup "rm -rf deno.lock" "deno i" "deno-this-pr i" "deno-this-pr i --unstable-lockfile-v5" Benchmark 1: deno i Time (mean ± σ): 464.4 ms ± 8.8 ms [User: 227.7 ms, System: 217.3 ms] Range (min … max): 452.6 ms … 478.3 ms 10 runs Benchmark 2: deno-this-pr i Time (mean ± σ): 368.8 ms ± 22.0 ms [User: 150.8 ms, System: 198.1 ms] Range (min … max): 344.8 ms … 397.6 ms 10 runs Benchmark 3: deno-this-pr i --unstable-lockfile-v5 Time (mean ± σ): 211.9 ms ± 17.1 ms [User: 7.1 ms, System: 177.2 ms] Range (min … max): 191.3 ms … 233.4 ms 10 runs Summary deno-this-pr i --unstable-lockfile-v5 ran 1.74 ± 0.17 times faster than deno-this-pr i 2.19 ± 0.18 times faster than deno i ``` With lockfile v5, a 2.19x faster install time compared to the current deno.
249 lines
7.1 KiB
Rust
249 lines
7.1 KiB
Rust
// Copyright 2018-2025 the Deno authors. MIT license.
|
|
|
|
use std::path::PathBuf;
|
|
use std::sync::Arc;
|
|
|
|
use deno_core::parking_lot::Mutex;
|
|
use deno_error::JsError;
|
|
use deno_error::JsErrorBox;
|
|
use deno_npm::registry::NpmRegistryApi;
|
|
use deno_npm::resolution::NpmResolutionSnapshot;
|
|
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
|
|
use deno_resolver::npm::managed::ManagedNpmResolverCreateOptions;
|
|
use deno_resolver::npm::managed::NpmResolutionCell;
|
|
use thiserror::Error;
|
|
|
|
use super::CliNpmRegistryInfoProvider;
|
|
use super::WorkspaceNpmPatchPackages;
|
|
use crate::args::CliLockfile;
|
|
use crate::sys::CliSys;
|
|
|
|
pub type CliManagedNpmResolverCreateOptions =
|
|
ManagedNpmResolverCreateOptions<CliSys>;
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub enum CliNpmResolverManagedSnapshotOption {
|
|
ResolveFromLockfile(Arc<CliLockfile>),
|
|
Specified(Option<ValidSerializedNpmResolutionSnapshot>),
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
enum SyncState {
|
|
Pending(Option<CliNpmResolverManagedSnapshotOption>),
|
|
Err(ResolveSnapshotError),
|
|
Success,
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
pub struct NpmResolutionInitializer {
|
|
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
|
|
npm_resolution: Arc<NpmResolutionCell>,
|
|
patch_packages: Arc<WorkspaceNpmPatchPackages>,
|
|
queue: tokio::sync::Mutex<()>,
|
|
sync_state: Mutex<SyncState>,
|
|
}
|
|
|
|
impl NpmResolutionInitializer {
|
|
pub fn new(
|
|
npm_registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
|
|
npm_resolution: Arc<NpmResolutionCell>,
|
|
patch_packages: Arc<WorkspaceNpmPatchPackages>,
|
|
snapshot_option: CliNpmResolverManagedSnapshotOption,
|
|
) -> Self {
|
|
Self {
|
|
npm_registry_info_provider,
|
|
npm_resolution,
|
|
patch_packages,
|
|
queue: tokio::sync::Mutex::new(()),
|
|
sync_state: Mutex::new(SyncState::Pending(Some(snapshot_option))),
|
|
}
|
|
}
|
|
|
|
#[cfg(debug_assertions)]
|
|
pub fn debug_assert_initialized(&self) {
|
|
if !matches!(*self.sync_state.lock(), SyncState::Success) {
|
|
panic!("debug assert: npm resolution must be initialized before calling this code");
|
|
}
|
|
}
|
|
|
|
pub async fn ensure_initialized(&self) -> Result<(), JsErrorBox> {
|
|
// fast exit if not pending
|
|
{
|
|
match &*self.sync_state.lock() {
|
|
SyncState::Pending(_) => {}
|
|
SyncState::Err(err) => return Err(JsErrorBox::from_err(err.clone())),
|
|
SyncState::Success => return Ok(()),
|
|
}
|
|
}
|
|
|
|
// only allow one task in here at a time
|
|
let _guard = self.queue.lock().await;
|
|
|
|
let snapshot_option = {
|
|
let mut sync_state = self.sync_state.lock();
|
|
match &mut *sync_state {
|
|
SyncState::Pending(snapshot_option) => {
|
|
// this should never panic, but if it does it means that a
|
|
// previous future was dropped while initialization occurred...
|
|
// that should never happen because this is initialized during
|
|
// startup
|
|
snapshot_option.take().unwrap()
|
|
}
|
|
// another thread updated the state while we were waiting
|
|
SyncState::Err(resolve_snapshot_error) => {
|
|
return Err(JsErrorBox::from_err(resolve_snapshot_error.clone()));
|
|
}
|
|
SyncState::Success => {
|
|
return Ok(());
|
|
}
|
|
}
|
|
};
|
|
|
|
match resolve_snapshot(
|
|
&self.npm_registry_info_provider,
|
|
snapshot_option,
|
|
&self.patch_packages,
|
|
)
|
|
.await
|
|
{
|
|
Ok(maybe_snapshot) => {
|
|
if let Some(snapshot) = maybe_snapshot {
|
|
self
|
|
.npm_resolution
|
|
.set_snapshot(NpmResolutionSnapshot::new(snapshot));
|
|
}
|
|
let mut sync_state = self.sync_state.lock();
|
|
*sync_state = SyncState::Success;
|
|
Ok(())
|
|
}
|
|
Err(err) => {
|
|
let mut sync_state = self.sync_state.lock();
|
|
*sync_state = SyncState::Err(err.clone());
|
|
Err(JsErrorBox::from_err(err))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Error, Clone, JsError)]
|
|
#[error("failed reading lockfile '{}'", lockfile_path.display())]
|
|
#[class(inherit)]
|
|
pub struct ResolveSnapshotError {
|
|
lockfile_path: PathBuf,
|
|
#[inherit]
|
|
#[source]
|
|
source: SnapshotFromLockfileError,
|
|
}
|
|
|
|
impl ResolveSnapshotError {
|
|
pub fn maybe_integrity_check_error(
|
|
&self,
|
|
) -> Option<&deno_npm::resolution::IntegrityCheckFailedError> {
|
|
match &self.source {
|
|
SnapshotFromLockfileError::SnapshotFromLockfile(
|
|
deno_npm::resolution::SnapshotFromLockfileError::IntegrityCheckFailed(
|
|
err,
|
|
),
|
|
) => Some(err),
|
|
_ => None,
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn resolve_snapshot(
|
|
registry_info_provider: &Arc<CliNpmRegistryInfoProvider>,
|
|
snapshot: CliNpmResolverManagedSnapshotOption,
|
|
patch_packages: &WorkspaceNpmPatchPackages,
|
|
) -> Result<Option<ValidSerializedNpmResolutionSnapshot>, ResolveSnapshotError>
|
|
{
|
|
match snapshot {
|
|
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(lockfile) => {
|
|
if !lockfile.overwrite() {
|
|
let snapshot = snapshot_from_lockfile(
|
|
lockfile.clone(),
|
|
®istry_info_provider.as_npm_registry_api(),
|
|
patch_packages,
|
|
)
|
|
.await
|
|
.map_err(|source| ResolveSnapshotError {
|
|
lockfile_path: lockfile.filename.clone(),
|
|
source,
|
|
})?;
|
|
Ok(Some(snapshot))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
CliNpmResolverManagedSnapshotOption::Specified(snapshot) => Ok(snapshot),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Error, Clone, JsError)]
|
|
pub enum SnapshotFromLockfileError {
|
|
// TODO(nathanwhit): remove once we've migrated to lockfile v5
|
|
#[error(transparent)]
|
|
#[class(inherit)]
|
|
IncompleteError(
|
|
#[from] deno_npm::resolution::IncompleteSnapshotFromLockfileError,
|
|
),
|
|
#[error(transparent)]
|
|
#[class(inherit)]
|
|
SnapshotFromLockfile(#[from] deno_npm::resolution::SnapshotFromLockfileError),
|
|
}
|
|
|
|
pub(crate) struct DefaultTarballUrl;
|
|
|
|
impl deno_npm::resolution::DefaultTarballUrlProvider for DefaultTarballUrl {
|
|
fn default_tarball_url(&self, id: &deno_npm::NpmPackageId) -> String {
|
|
let scope = id.nv.scope();
|
|
let package_name = if let Some(scope) = scope {
|
|
id.nv
|
|
.name
|
|
.strip_prefix(scope)
|
|
.unwrap_or(&id.nv.name)
|
|
.trim_start_matches('/')
|
|
} else {
|
|
&id.nv.name
|
|
};
|
|
format!(
|
|
"https://registry.npmjs.org/{}/-/{}-{}.tgz",
|
|
id.nv.name, package_name, id.nv.version
|
|
)
|
|
}
|
|
}
|
|
|
|
async fn snapshot_from_lockfile(
|
|
lockfile: Arc<CliLockfile>,
|
|
api: &dyn NpmRegistryApi,
|
|
patch_packages: &WorkspaceNpmPatchPackages,
|
|
) -> Result<ValidSerializedNpmResolutionSnapshot, SnapshotFromLockfileError> {
|
|
if lockfile.use_lockfile_v5() {
|
|
let snapshot = deno_npm::resolution::snapshot_from_lockfile_v5(
|
|
deno_npm::resolution::SnapshotFromLockfileV5Params {
|
|
patch_packages: &patch_packages.0,
|
|
lockfile: &lockfile.lock(),
|
|
default_tarball_url: &DefaultTarballUrl,
|
|
},
|
|
)?;
|
|
|
|
Ok(snapshot)
|
|
} else {
|
|
let (incomplete_snapshot, skip_integrity_check) = {
|
|
let lock = lockfile.lock();
|
|
(
|
|
deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?,
|
|
lock.overwrite,
|
|
)
|
|
};
|
|
let snapshot = deno_npm::resolution::snapshot_from_lockfile(
|
|
deno_npm::resolution::SnapshotFromLockfileParams {
|
|
incomplete_snapshot,
|
|
api,
|
|
patch_packages: &patch_packages.0,
|
|
skip_integrity_check,
|
|
},
|
|
)
|
|
.await?;
|
|
Ok(snapshot)
|
|
}
|
|
}
|