deno/cli/npm/installer/common/bin_entries.rs
Nathan Whitaker 780b741555
perf(npm): load npm resolution snapshot directly from lockfile (#28647)
Fixes #27264. Fixes https://github.com/denoland/deno/issues/28161.

Currently the new lockfile version is gated behind an unstable flag
(`--unstable-lockfile-v5`) until the next minor release, where it will
become the default.

The main motivation here is that it improves startup performance when
using the global cache or `--node-modules-dir=auto`.

In a create-next-app project, running an empty file:
```
❯ hyperfine --warmup 25 -N --setup "rm -f deno.lock" "deno run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto -A empty.js" "deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js" "deno run --node-modules-dir=manual -A empty.js" "deno-this-pr run --node-modules-dir=manual -A empty.js"
Benchmark 1: deno run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     247.6 ms ±   1.7 ms    [User: 228.7 ms, System: 19.0 ms]
  Range (min … max):   245.5 ms … 251.5 ms    12 runs

Benchmark 2: deno-this-pr run --node-modules-dir=auto -A empty.js
  Time (mean ± σ):     169.8 ms ±   1.0 ms    [User: 152.9 ms, System: 17.9 ms]
  Range (min … max):   168.9 ms … 172.5 ms    17 runs

Benchmark 3: deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
  Time (mean ± σ):      16.2 ms ±   0.7 ms    [User: 12.3 ms, System: 5.7 ms]
  Range (min … max):    15.2 ms …  19.2 ms    185 runs

Benchmark 4: deno run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.2 ms ±   0.8 ms    [User: 11.6 ms, System: 5.5 ms]
  Range (min … max):    14.9 ms …  19.7 ms    187 runs

Benchmark 5: deno-this-pr run --node-modules-dir=manual -A empty.js
  Time (mean ± σ):      16.0 ms ±   0.9 ms    [User: 12.0 ms, System: 5.5 ms]
  Range (min … max):    14.8 ms …  22.3 ms    190 runs

  Warning: Statistical outliers were detected. Consider re-running this benchmark on a quiet system without any interferences from other programs. It might help to use the '--warmup' or '--prepare' options.

Summary
  deno-this-pr run --node-modules-dir=manual -A empty.js ran
    1.01 ± 0.08 times faster than deno run --node-modules-dir=manual -A empty.js
    1.01 ± 0.07 times faster than deno-this-pr run --node-modules-dir=auto --unstable-lockfile-v5 empty.js
   10.64 ± 0.60 times faster than deno-this-pr run --node-modules-dir=auto -A empty.js
   15.51 ± 0.88 times faster than deno run --node-modules-dir=auto -A empty.js
```

When using the new lockfile version, this leads to a 15.5x faster
startup time compared to the current deno version.

Install times benefit as well, though to a lesser degree.

`deno install` on a create-next-app project, with everything cached
(just setting up node_modules from scratch):

```
❯ hyperfine --warmup 5 -N --prepare "rm -rf node_modules" --setup "rm -rf deno.lock" "deno i" "deno-this-pr i" "deno-this-pr i --unstable-lockfile-v5"
Benchmark 1: deno i
  Time (mean ± σ):     464.4 ms ±   8.8 ms    [User: 227.7 ms, System: 217.3 ms]
  Range (min … max):   452.6 ms … 478.3 ms    10 runs

Benchmark 2: deno-this-pr i
  Time (mean ± σ):     368.8 ms ±  22.0 ms    [User: 150.8 ms, System: 198.1 ms]
  Range (min … max):   344.8 ms … 397.6 ms    10 runs

Benchmark 3: deno-this-pr i --unstable-lockfile-v5
  Time (mean ± σ):     211.9 ms ±  17.1 ms    [User: 7.1 ms, System: 177.2 ms]
  Range (min … max):   191.3 ms … 233.4 ms    10 runs

Summary
  deno-this-pr i --unstable-lockfile-v5 ran
    1.74 ± 0.17 times faster than deno-this-pr i
    2.19 ± 0.18 times faster than deno i
```

With lockfile v5, a 2.19x faster install time compared to the current
deno.
2025-04-08 02:06:17 +00:00

510 lines
13 KiB
Rust

// Copyright 2018-2025 the Deno authors. MIT license.
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use std::path::Path;
use std::path::PathBuf;
use deno_npm::resolution::NpmResolutionSnapshot;
use deno_npm::NpmPackageExtraInfo;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
#[derive(Default)]
pub struct BinEntries<'a> {
/// Packages that have colliding bin names
collisions: HashSet<&'a NpmPackageId>,
seen_names: HashMap<String, &'a NpmPackageId>,
/// The bin entries
entries: Vec<(&'a NpmResolutionPackage, PathBuf, NpmPackageExtraInfo)>,
sorted: bool,
}
/// Returns the name of the default binary for the given package.
/// This is the package name without the organization (`@org/`), if any.
fn default_bin_name(package: &NpmResolutionPackage) -> &str {
package
.id
.nv
.name
.as_str()
.rsplit_once('/')
.map(|(_, name)| name)
.unwrap_or(package.id.nv.name.as_str())
}
pub fn warn_missing_entrypoint(
bin_name: &str,
package_path: &Path,
entrypoint: &Path,
) {
log::warn!(
"{} Trying to set up '{}' bin for \"{}\", but the entry point \"{}\" doesn't exist.",
deno_terminal::colors::yellow("Warning"),
bin_name,
package_path.display(),
entrypoint.display()
);
}
#[derive(Debug, thiserror::Error, deno_error::JsError)]
pub enum BinEntriesError {
#[class(inherit)]
#[error("Creating '{path}'")]
Creating {
path: PathBuf,
#[source]
#[inherit]
source: std::io::Error,
},
#[cfg(unix)]
#[class(inherit)]
#[error("Setting permissions on '{path}'")]
Permissions {
path: PathBuf,
#[source]
#[inherit]
source: std::io::Error,
},
#[class(inherit)]
#[error("Can't set up '{name}' bin at {path}")]
SetUpBin {
name: String,
path: PathBuf,
#[source]
#[inherit]
source: Box<Self>,
},
#[cfg(unix)]
#[class(inherit)]
#[error("Setting permissions on '{path}'")]
RemoveBinSymlink {
path: PathBuf,
#[source]
#[inherit]
source: std::io::Error,
},
#[class(inherit)]
#[error(transparent)]
Io(#[from] std::io::Error),
}
impl<'a> BinEntries<'a> {
pub fn new() -> Self {
Self::default()
}
/// Add a new bin entry (package with a bin field)
pub fn add<'b>(
&mut self,
package: &'a NpmResolutionPackage,
extra: &'b NpmPackageExtraInfo,
package_path: PathBuf,
) {
self.sorted = false;
// check for a new collision, if we haven't already
// found one
match extra.bin.as_ref().unwrap() {
deno_npm::registry::NpmPackageVersionBinEntry::String(_) => {
let bin_name = default_bin_name(package);
if let Some(other) =
self.seen_names.insert(bin_name.to_string(), &package.id)
{
self.collisions.insert(&package.id);
self.collisions.insert(other);
}
}
deno_npm::registry::NpmPackageVersionBinEntry::Map(entries) => {
for name in entries.keys() {
if let Some(other) =
self.seen_names.insert(name.to_string(), &package.id)
{
self.collisions.insert(&package.id);
self.collisions.insert(other);
}
}
}
}
self.entries.push((package, package_path, extra.clone()));
}
fn for_each_entry(
&mut self,
snapshot: &NpmResolutionSnapshot,
mut already_seen: impl FnMut(
&Path,
&str, // bin script
) -> Result<(), BinEntriesError>,
mut new: impl FnMut(
&NpmResolutionPackage,
&NpmPackageExtraInfo,
&Path,
&str, // bin name
&str, // bin script
) -> Result<(), BinEntriesError>,
mut filter: impl FnMut(&NpmResolutionPackage) -> bool,
) -> Result<(), BinEntriesError> {
if !self.collisions.is_empty() && !self.sorted {
// walking the dependency tree to find out the depth of each package
// is sort of expensive, so we only do it if there's a collision
sort_by_depth(snapshot, &mut self.entries, &mut self.collisions);
self.sorted = true;
}
let mut seen = HashSet::new();
for (package, package_path, extra) in &self.entries {
if !filter(package) {
continue;
}
if let Some(bin_entries) = &extra.bin {
match bin_entries {
deno_npm::registry::NpmPackageVersionBinEntry::String(script) => {
let name = default_bin_name(package);
if !seen.insert(name) {
already_seen(package_path, script)?;
// we already set up a bin entry with this name
continue;
}
new(package, extra, package_path, name, script)?;
}
deno_npm::registry::NpmPackageVersionBinEntry::Map(entries) => {
for (name, script) in entries {
if !seen.insert(name) {
already_seen(package_path, script)?;
// we already set up a bin entry with this name
continue;
}
new(package, extra, package_path, name, script)?;
}
}
}
}
}
Ok(())
}
/// Collect the bin entries into a vec of (name, script path)
pub fn collect_bin_files(
&mut self,
snapshot: &NpmResolutionSnapshot,
) -> Vec<(String, PathBuf)> {
let mut bins = Vec::new();
self
.for_each_entry(
snapshot,
|_, _| Ok(()),
|_, _, package_path, name, script| {
bins.push((name.to_string(), package_path.join(script)));
Ok(())
},
|_| true,
)
.unwrap();
bins
}
fn set_up_entries_filtered(
mut self,
snapshot: &NpmResolutionSnapshot,
bin_node_modules_dir_path: &Path,
filter: impl FnMut(&NpmResolutionPackage) -> bool,
mut handler: impl FnMut(&EntrySetupOutcome<'_>),
) -> Result<(), BinEntriesError> {
if !self.entries.is_empty() && !bin_node_modules_dir_path.exists() {
std::fs::create_dir_all(bin_node_modules_dir_path).map_err(|source| {
BinEntriesError::Creating {
path: bin_node_modules_dir_path.to_path_buf(),
source,
}
})?;
}
self.for_each_entry(
snapshot,
|_package_path, _script| {
#[cfg(unix)]
{
let path = _package_path.join(_script);
make_executable_if_exists(&path)?;
}
Ok(())
},
|package, extra, package_path, name, script| {
let outcome = set_up_bin_entry(
package,
extra,
name,
script,
package_path,
bin_node_modules_dir_path,
)?;
handler(&outcome);
Ok(())
},
filter,
)?;
Ok(())
}
/// Finish setting up the bin entries, writing the necessary files
/// to disk.
pub fn finish(
self,
snapshot: &NpmResolutionSnapshot,
bin_node_modules_dir_path: &Path,
handler: impl FnMut(&EntrySetupOutcome<'_>),
) -> Result<(), BinEntriesError> {
self.set_up_entries_filtered(
snapshot,
bin_node_modules_dir_path,
|_| true,
handler,
)
}
/// Finish setting up the bin entries, writing the necessary files
/// to disk.
pub fn finish_only(
self,
snapshot: &NpmResolutionSnapshot,
bin_node_modules_dir_path: &Path,
handler: impl FnMut(&EntrySetupOutcome<'_>),
only: &HashSet<&NpmPackageId>,
) -> Result<(), BinEntriesError> {
self.set_up_entries_filtered(
snapshot,
bin_node_modules_dir_path,
|package| only.contains(&package.id),
handler,
)
}
}
// walk the dependency tree to find out the depth of each package
// that has a bin entry, then sort them by depth
fn sort_by_depth(
snapshot: &NpmResolutionSnapshot,
bin_entries: &mut [(&NpmResolutionPackage, PathBuf, NpmPackageExtraInfo)],
collisions: &mut HashSet<&NpmPackageId>,
) {
enum Entry<'a> {
Pkg(&'a NpmPackageId),
IncreaseDepth,
}
let mut seen = HashSet::new();
let mut depths: HashMap<&NpmPackageId, u64> =
HashMap::with_capacity(collisions.len());
let mut queue = VecDeque::new();
queue.extend(snapshot.top_level_packages().map(Entry::Pkg));
seen.extend(snapshot.top_level_packages());
queue.push_back(Entry::IncreaseDepth);
let mut current_depth = 0u64;
while let Some(entry) = queue.pop_front() {
if collisions.is_empty() {
break;
}
let id = match entry {
Entry::Pkg(id) => id,
Entry::IncreaseDepth => {
current_depth += 1;
if queue.is_empty() {
break;
}
queue.push_back(Entry::IncreaseDepth);
continue;
}
};
if let Some(package) = snapshot.package_from_id(id) {
if collisions.remove(&package.id) {
depths.insert(&package.id, current_depth);
}
for dep in package.dependencies.values() {
if seen.insert(dep) {
queue.push_back(Entry::Pkg(dep));
}
}
}
}
bin_entries.sort_by(|(a, _, _), (b, _, _)| {
depths
.get(&a.id)
.unwrap_or(&u64::MAX)
.cmp(depths.get(&b.id).unwrap_or(&u64::MAX))
.then_with(|| a.id.nv.cmp(&b.id.nv).reverse())
});
}
pub fn set_up_bin_entry<'a>(
package: &'a NpmResolutionPackage,
#[allow(unused_variables)] extra: &'a NpmPackageExtraInfo,
bin_name: &'a str,
#[allow(unused_variables)] bin_script: &str,
#[allow(unused_variables)] package_path: &'a Path,
bin_node_modules_dir_path: &Path,
) -> Result<EntrySetupOutcome<'a>, BinEntriesError> {
#[cfg(windows)]
{
set_up_bin_shim(package, bin_name, bin_node_modules_dir_path)?;
Ok(EntrySetupOutcome::Success)
}
#[cfg(unix)]
{
symlink_bin_entry(
package,
extra,
bin_name,
bin_script,
package_path,
bin_node_modules_dir_path,
)
}
}
#[cfg(windows)]
fn set_up_bin_shim(
package: &NpmResolutionPackage,
bin_name: &str,
bin_node_modules_dir_path: &Path,
) -> Result<(), BinEntriesError> {
use std::fs;
let mut cmd_shim = bin_node_modules_dir_path.join(bin_name);
cmd_shim.set_extension("cmd");
let shim = format!("@deno run -A npm:{}/{bin_name} %*", package.id.nv);
fs::write(&cmd_shim, shim).map_err(|err| BinEntriesError::SetUpBin {
name: bin_name.to_string(),
path: cmd_shim.clone(),
source: Box::new(err.into()),
})?;
Ok(())
}
#[cfg(unix)]
/// Make the file at `path` executable if it exists.
/// Returns `true` if the file exists, `false` otherwise.
fn make_executable_if_exists(path: &Path) -> Result<bool, BinEntriesError> {
use std::io;
use std::os::unix::fs::PermissionsExt;
let mut perms = match std::fs::metadata(path) {
Ok(metadata) => metadata.permissions(),
Err(err) => {
if err.kind() == io::ErrorKind::NotFound {
return Ok(false);
}
return Err(err.into());
}
};
if perms.mode() & 0o111 == 0 {
// if the original file is not executable, make it executable
perms.set_mode(perms.mode() | 0o111);
std::fs::set_permissions(path, perms).map_err(|source| {
BinEntriesError::Permissions {
path: path.to_path_buf(),
source,
}
})?;
}
Ok(true)
}
pub enum EntrySetupOutcome<'a> {
#[cfg_attr(windows, allow(dead_code))]
MissingEntrypoint {
bin_name: &'a str,
package_path: &'a Path,
entrypoint: PathBuf,
package: &'a NpmResolutionPackage,
extra: &'a NpmPackageExtraInfo,
},
Success,
}
impl EntrySetupOutcome<'_> {
pub fn warn_if_failed(&self) {
match self {
EntrySetupOutcome::MissingEntrypoint {
bin_name,
package_path,
entrypoint,
..
} => warn_missing_entrypoint(bin_name, package_path, entrypoint),
EntrySetupOutcome::Success => {}
}
}
}
#[cfg(unix)]
fn symlink_bin_entry<'a>(
package: &'a NpmResolutionPackage,
extra: &'a NpmPackageExtraInfo,
bin_name: &'a str,
bin_script: &str,
package_path: &'a Path,
bin_node_modules_dir_path: &Path,
) -> Result<EntrySetupOutcome<'a>, BinEntriesError> {
use std::io;
use std::os::unix::fs::symlink;
let link = bin_node_modules_dir_path.join(bin_name);
let original = package_path.join(bin_script);
let found = make_executable_if_exists(&original).map_err(|source| {
BinEntriesError::SetUpBin {
name: bin_name.to_string(),
path: original.to_path_buf(),
source: Box::new(source),
}
})?;
if !found {
return Ok(EntrySetupOutcome::MissingEntrypoint {
bin_name,
package_path,
entrypoint: original,
package,
extra,
});
}
let original_relative =
crate::util::path::relative_path(bin_node_modules_dir_path, &original)
.unwrap_or(original);
if let Err(err) = symlink(&original_relative, &link) {
if err.kind() == io::ErrorKind::AlreadyExists {
// remove and retry
std::fs::remove_file(&link).map_err(|source| {
BinEntriesError::RemoveBinSymlink {
path: link.clone(),
source,
}
})?;
symlink(&original_relative, &link).map_err(|source| {
BinEntriesError::SetUpBin {
name: bin_name.to_string(),
path: original_relative.to_path_buf(),
source: Box::new(source.into()),
}
})?;
return Ok(EntrySetupOutcome::Success);
}
return Err(BinEntriesError::SetUpBin {
name: bin_name.to_string(),
path: original_relative.to_path_buf(),
source: Box::new(err.into()),
});
}
Ok(EntrySetupOutcome::Success)
}