language-servers/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-08-04 10:59:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refactor: add deno_npm_cache crate (#27200)

Browse source 
Extracting out more code from the CLI for reuse elsewhere (still more
work to do, but this is a start).

This is the code for extracting npm tarballs and saving information in
the npm cache in the global deno_dir.
This commit is contained in:
David Sherret 2024-12-02 21:10:16 -05:00 • committed by Bartek Iwańczuk
parent a3a8cc4129
commit 774232764b
No known key found for this signature in database
GPG key ID: 0C6BCDDC3B3AD750
22 changed files with 492 additions and 435 deletions
Download patch file Download diff file Expand all files Collapse all files

59
cli/npm/common.rs
View file

@ -1,59 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_npm::npm_rc::RegistryConfig;
use http::header;
// TODO(bartlomieju): support more auth methods besides token and basic auth
pub fn maybe_auth_header_for_npm_registry(
registry_config: &RegistryConfig,
) -> Result<Option<(header::HeaderName, header::HeaderValue)>, AnyError> {
if let Some(token) = registry_config.auth_token.as_ref() {
return Ok(Some((
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!("Bearer {}", token)).unwrap(),
)));
}
if let Some(auth) = registry_config.auth.as_ref() {
return Ok(Some((
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!("Basic {}", auth)).unwrap(),
)));
}
let (username, password) = (
registry_config.username.as_ref(),
registry_config.password.as_ref(),
);
if (username.is_some() && password.is_none())
|| (username.is_none() && password.is_some())
{
bail!("Both the username and password must be provided for basic auth")
}
if username.is_some() && password.is_some() {
// The npm client does some double encoding when generating the
// bearer token value, see
// https://github.com/npm/cli/blob/780afc50e3a345feb1871a28e33fa48235bc3bd5/workspaces/config/lib/index.js#L846-L851
let pw_base64 = BASE64_STANDARD
.decode(password.unwrap())
.with_context(|| "The password in npmrc is an invalid base64 string")?;
let bearer = BASE64_STANDARD.encode(format!(
"{}:{}",
username.unwrap(),
String::from_utf8_lossy(&pw_base64)
));
return Ok(Some((
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!("Basic {}", bearer)).unwrap(),
)));
}
Ok(None)
}

280
cli/npm/managed/cache/mod.rs vendored
View file

@ -1,280 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use crate::args::CacheSetting;
use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::hard_link_dir_recursive;
pub mod registry_info;
mod tarball;
mod tarball_extract;
pub use registry_info::RegistryInfoDownloader;
pub use tarball::TarballCache;
/// Stores a single copy of npm packages in a cache.
#[derive(Debug)]
pub struct NpmCache {
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
npmrc: Arc<ResolvedNpmRc>,
/// ensures a package is only downloaded once per run
previously_reloaded_packages: Mutex<HashSet<PackageNv>>,
}
impl NpmCache {
pub fn new(
cache_dir: Arc<NpmCacheDir>,
cache_setting: CacheSetting,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
cache_dir,
cache_setting,
previously_reloaded_packages: Default::default(),
npmrc,
}
}
pub fn cache_setting(&self) -> &CacheSetting {
&self.cache_setting
}
pub fn root_dir_path(&self) -> &Path {
self.cache_dir.root_dir()
}
pub fn root_dir_url(&self) -> &Url {
self.cache_dir.root_dir_url()
}
/// Checks if the cache should be used for the provided name and version.
/// NOTE: Subsequent calls for the same package will always return `true`
/// to ensure a package is only downloaded once per run of the CLI. This
/// prevents downloads from re-occurring when someone has `--reload` and
/// and imports a dynamic import that imports the same package again for example.
pub fn should_use_cache_for_package(&self, package: &PackageNv) -> bool {
self.cache_setting.should_use_for_npm_package(&package.name)
|| !self
.previously_reloaded_packages
.lock()
.insert(package.clone())
}
/// Ensures a copy of the package exists in the global cache.
///
/// This assumes that the original package folder being hard linked
/// from exists before this is called.
pub fn ensure_copy_package(
&self,
folder_id: &NpmPackageCacheFolderId,
) -> Result<(), AnyError> {
let registry_url = self.npmrc.get_registry_url(&folder_id.nv.name);
assert_ne!(folder_id.copy_index, 0);
let package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
folder_id.copy_index,
registry_url,
);
if package_folder.exists()
// if this file exists, then the package didn't successfully initialize
// the first time, or another process is currently extracting the zip file
&& !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists()
&& self.cache_setting.should_use_for_npm_package(&folder_id.nv.name)
{
return Ok(());
}
let original_package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
0, // original copy index
registry_url,
);
// it seems Windows does an "AccessDenied" error when moving a
// directory with hard links, so that's why this solution is done
with_folder_sync_lock(&folder_id.nv, &package_folder, || {
hard_link_dir_recursive(&original_package_folder, &package_folder)
})?;
Ok(())
}
pub fn package_folder_for_id(&self, id: &NpmPackageCacheFolderId) -> PathBuf {
let registry_url = self.npmrc.get_registry_url(&id.nv.name);
self.cache_dir.package_folder_for_id(
&id.nv.name,
&id.nv.version.to_string(),
id.copy_index,
registry_url,
)
}
pub fn package_folder_for_nv(&self, package: &PackageNv) -> PathBuf {
let registry_url = self.npmrc.get_registry_url(&package.name);
self.package_folder_for_nv_and_url(package, registry_url)
}
pub fn package_folder_for_nv_and_url(
&self,
package: &PackageNv,
registry_url: &Url,
) -> PathBuf {
self.cache_dir.package_folder_for_id(
&package.name,
&package.version.to_string(),
0, // original copy_index
registry_url,
)
}
pub fn package_name_folder(&self, name: &str) -> PathBuf {
let registry_url = self.npmrc.get_registry_url(name);
self.cache_dir.package_name_folder(name, registry_url)
}
pub fn resolve_package_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Option<NpmPackageCacheFolderId> {
self
.cache_dir
.resolve_package_folder_id_from_specifier(specifier)
.and_then(|cache_id| {
Some(NpmPackageCacheFolderId {
nv: PackageNv {
name: cache_id.name,
version: Version::parse_from_npm(&cache_id.version).ok()?,
},
copy_index: cache_id.copy_index,
})
})
}
pub fn load_package_info(
&self,
name: &str,
) -> Result<Option<NpmPackageInfo>, AnyError> {
let file_cache_path = self.get_registry_package_info_file_cache_path(name);
let file_text = match fs::read_to_string(file_cache_path) {
Ok(file_text) => file_text,
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
Err(err) => return Err(err.into()),
};
Ok(serde_json::from_str(&file_text)?)
}
pub fn save_package_info(
&self,
name: &str,
package_info: &NpmPackageInfo,
) -> Result<(), AnyError> {
let file_cache_path = self.get_registry_package_info_file_cache_path(name);
let file_text = serde_json::to_string(&package_info)?;
atomic_write_file_with_retries(&file_cache_path, file_text, CACHE_PERM)?;
Ok(())
}
fn get_registry_package_info_file_cache_path(&self, name: &str) -> PathBuf {
let name_folder_path = self.package_name_folder(name);
name_folder_path.join("registry.json")
}
}
const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock";
fn with_folder_sync_lock(
package: &PackageNv,
output_folder: &Path,
action: impl FnOnce() -> Result<(), AnyError>,
) -> Result<(), AnyError> {
fn inner(
output_folder: &Path,
action: impl FnOnce() -> Result<(), AnyError>,
) -> Result<(), AnyError> {
fs::create_dir_all(output_folder).with_context(|| {
format!("Error creating '{}'.", output_folder.display())
})?;
// This sync lock file is a way to ensure that partially created
// npm package directories aren't considered valid. This could maybe
// be a bit smarter in the future to not bother extracting here
// if another process has taken the lock in the past X seconds and
// wait for the other process to finish (it could try to create the
// file with `create_new(true)` then if it exists, check the metadata
// then wait until the other process finishes with a timeout), but
// for now this is good enough.
let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME);
match fs::OpenOptions::new()
.write(true)
.create(true)
.truncate(false)
.open(&sync_lock_path)
{
Ok(_) => {
action()?;
// extraction succeeded, so only now delete this file
let _ignore = std::fs::remove_file(&sync_lock_path);
Ok(())
}
Err(err) => {
bail!(
concat!(
"Error creating package sync lock file at '{}'. ",
"Maybe try manually deleting this folder.\n\n{:#}",
),
output_folder.display(),
err
);
}
}
}
match inner(output_folder, action) {
Ok(()) => Ok(()),
Err(err) => {
if let Err(remove_err) = fs::remove_dir_all(output_folder) {
if remove_err.kind() != std::io::ErrorKind::NotFound {
bail!(
concat!(
"Failed setting up package cache directory for {}, then ",
"failed cleaning it up.\n\nOriginal error:\n\n{}\n\n",
"Remove error:\n\n{}\n\nPlease manually ",
"delete this folder or you will run into issues using this ",
"package in the future:\n\n{}"
),
package,
err,
remove_err,
output_folder.display(),
);
}
}
Err(err)
}
}
}

274
cli/npm/managed/cache/registry_info.rs vendored
View file

@ -1,274 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashMap;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures::future::LocalBoxFuture;
use deno_core::futures::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use crate::args::CacheSetting;
use crate::http_util::HttpClientProvider;
use crate::npm::common::maybe_auth_header_for_npm_registry;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::MultiRuntimeAsyncValueCreator;
use super::NpmCache;
// todo(dsherret): create seams and unit test this
type LoadResult = Result<FutureResult, Arc<AnyError>>;
type LoadFuture = LocalBoxFuture<'static, LoadResult>;
#[derive(Debug, Clone)]
enum FutureResult {
PackageNotExists,
SavedFsCache(Arc<NpmPackageInfo>),
ErroredFsCache(Arc<NpmPackageInfo>),
}
#[derive(Debug, Clone)]
enum MemoryCacheItem {
/// The cache item hasn't loaded yet.
Pending(Arc<MultiRuntimeAsyncValueCreator<LoadResult>>),
/// The item has loaded in the past and was stored in the file system cache.
/// There is no reason to request this package from the npm registry again
/// for the duration of execution.
FsCached,
/// An item is memory cached when it fails saving to the file system cache
/// or the package does not exist.
MemoryCached(Result<Option<Arc<NpmPackageInfo>>, Arc<AnyError>>),
}
/// Downloads packuments from the npm registry.
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct RegistryInfoDownloader {
cache: Arc<NpmCache>,
http_client_provider: Arc<HttpClientProvider>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
memory_cache: Mutex<HashMap<String, MemoryCacheItem>>,
}
impl RegistryInfoDownloader {
pub fn new(
cache: Arc<NpmCache>,
http_client_provider: Arc<HttpClientProvider>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
) -> Self {
Self {
cache,
http_client_provider,
npmrc,
progress_bar,
memory_cache: Default::default(),
}
}
pub async fn load_package_info(
self: &Arc<Self>,
name: &str,
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
self.load_package_info_inner(name).await.with_context(|| {
format!(
"Error getting response at {} for package \"{}\"",
get_package_url(&self.npmrc, name),
name
)
})
}
async fn load_package_info_inner(
self: &Arc<Self>,
name: &str,
) -> Result<Option<Arc<NpmPackageInfo>>, AnyError> {
if *self.cache.cache_setting() == CacheSetting::Only {
return Err(custom_error(
"NotCached",
format!(
"An npm specifier not found in cache: \"{name}\", --cached-only is specified."
)
));
}
let cache_item = {
let mut mem_cache = self.memory_cache.lock();
if let Some(cache_item) = mem_cache.get(name) {
cache_item.clone()
} else {
let value_creator = MultiRuntimeAsyncValueCreator::new({
let downloader = self.clone();
let name = name.to_string();
Box::new(move || downloader.create_load_future(&name))
});
let cache_item = MemoryCacheItem::Pending(Arc::new(value_creator));
mem_cache.insert(name.to_string(), cache_item.clone());
cache_item
}
};
match cache_item {
MemoryCacheItem::FsCached => {
// this struct previously loaded from the registry, so we can load it from the file system cache
self
.load_file_cached_package_info(name)
.await
.map(|info| Some(Arc::new(info)))
}
MemoryCacheItem::MemoryCached(maybe_info) => {
maybe_info.clone().map_err(|e| anyhow!("{}", e))
}
MemoryCacheItem::Pending(value_creator) => {
match value_creator.get().await {
Ok(FutureResult::SavedFsCache(info)) => {
// return back the future and mark this package as having
// been saved in the cache for next time it's requested
*self.memory_cache.lock().get_mut(name).unwrap() =
MemoryCacheItem::FsCached;
Ok(Some(info))
}
Ok(FutureResult::ErroredFsCache(info)) => {
// since saving to the fs cache failed, keep the package information in memory
*self.memory_cache.lock().get_mut(name).unwrap() =
MemoryCacheItem::MemoryCached(Ok(Some(info.clone())));
Ok(Some(info))
}
Ok(FutureResult::PackageNotExists) => {
*self.memory_cache.lock().get_mut(name).unwrap() =
MemoryCacheItem::MemoryCached(Ok(None));
Ok(None)
}
Err(err) => {
let return_err = anyhow!("{}", err);
*self.memory_cache.lock().get_mut(name).unwrap() =
MemoryCacheItem::MemoryCached(Err(err));
Err(return_err)
}
}
}
}
}
async fn load_file_cached_package_info(
&self,
name: &str,
) -> Result<NpmPackageInfo, AnyError> {
// this scenario failing should be exceptionally rare so let's
// deal with improving it only when anyone runs into an issue
let maybe_package_info = deno_core::unsync::spawn_blocking({
let cache = self.cache.clone();
let name = name.to_string();
move || cache.load_package_info(&name)
})
.await
.unwrap()
.with_context(|| {
format!(
"Previously saved '{}' from the npm cache, but now it fails to load.",
name
)
})?;
match maybe_package_info {
Some(package_info) => Ok(package_info),
None => {
bail!("The package '{}' previously saved its registry information to the file system cache, but that file no longer exists.", name)
}
}
}
fn create_load_future(self: &Arc<Self>, name: &str) -> LoadFuture {
let downloader = self.clone();
let package_url = get_package_url(&self.npmrc, name);
let registry_config = self.npmrc.get_registry_config(name);
let maybe_auth_header =
match maybe_auth_header_for_npm_registry(registry_config) {
Ok(maybe_auth_header) => maybe_auth_header,
Err(err) => {
return std::future::ready(Err(Arc::new(err))).boxed_local()
}
};
let guard = self.progress_bar.update(package_url.as_str());
let name = name.to_string();
async move {
let client = downloader.http_client_provider.get_or_create()?;
let maybe_bytes = client
.download_with_progress_and_retries(
package_url,
maybe_auth_header,
&guard,
)
.await?;
match maybe_bytes {
Some(bytes) => {
let future_result = deno_core::unsync::spawn_blocking(
move || -> Result<FutureResult, AnyError> {
let package_info = serde_json::from_slice(&bytes)?;
match downloader.cache.save_package_info(&name, &package_info) {
Ok(()) => {
Ok(FutureResult::SavedFsCache(Arc::new(package_info)))
}
Err(err) => {
log::debug!(
"Error saving package {} to cache: {:#}",
name,
err
);
Ok(FutureResult::ErroredFsCache(Arc::new(package_info)))
}
}
},
)
.await??;
Ok(future_result)
}
None => Ok(FutureResult::PackageNotExists),
}
}
.map(|r| r.map_err(Arc::new))
.boxed_local()
}
}
pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url {
let registry_url = npmrc.get_registry_url(name);
// The '/' character in scoped package names "@scope/name" must be
// encoded for older third party registries. Newer registries and
// npm itself support both ways
// - encoded: https://registry.npmjs.org/@rollup%2fplugin-json
// - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json
// To support as many third party registries as possible we'll
// always encode the '/' character.
// list of all characters used in npm packages:
// !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~
const ASCII_SET: percent_encoding::AsciiSet =
percent_encoding::NON_ALPHANUMERIC
.remove(b'!')
.remove(b'\'')
.remove(b'(')
.remove(b')')
.remove(b'*')
.remove(b'-')
.remove(b'.')
.remove(b'@')
.remove(b'_')
.remove(b'~');
let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET);
registry_url
// Ensure that scoped package name percent encoding is lower cased
// to match npm.
.join(&name.to_string().replace("%2F", "%2f"))
.unwrap()
}

235
cli/npm/managed/cache/tarball.rs vendored
View file

@ -1,235 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashMap;
use std::sync::Arc;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures::future::LocalBoxFuture;
use deno_core::futures::FutureExt;
use deno_core::parking_lot::Mutex;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageVersionDistInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_semver::package::PackageNv;
use http::StatusCode;
use crate::args::CacheSetting;
use crate::http_util::DownloadError;
use crate::http_util::HttpClientProvider;
use crate::npm::common::maybe_auth_header_for_npm_registry;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::MultiRuntimeAsyncValueCreator;
use super::tarball_extract::verify_and_extract_tarball;
use super::tarball_extract::TarballExtractionMode;
use super::NpmCache;
// todo(dsherret): create seams and unit test this
type LoadResult = Result<(), Arc<AnyError>>;
type LoadFuture = LocalBoxFuture<'static, LoadResult>;
#[derive(Debug, Clone)]
enum MemoryCacheItem {
/// The cache item hasn't finished yet.
Pending(Arc<MultiRuntimeAsyncValueCreator<LoadResult>>),
/// The result errored.
Errored(Arc<AnyError>),
/// This package has already been cached.
Cached,
}
/// Coordinates caching of tarballs being loaded from
/// the npm registry.
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct TarballCache {
cache: Arc<NpmCache>,
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
memory_cache: Mutex<HashMap<PackageNv, MemoryCacheItem>>,
}
impl TarballCache {
pub fn new(
cache: Arc<NpmCache>,
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
npmrc: Arc<ResolvedNpmRc>,
progress_bar: ProgressBar,
) -> Self {
Self {
cache,
fs,
http_client_provider,
npmrc,
progress_bar,
memory_cache: Default::default(),
}
}
pub async fn ensure_package(
self: &Arc<Self>,
package: &PackageNv,
dist: &NpmPackageVersionDistInfo,
) -> Result<(), AnyError> {
self
.ensure_package_inner(package, dist)
.await
.with_context(|| format!("Failed caching npm package '{}'.", package))
}
async fn ensure_package_inner(
self: &Arc<Self>,
package_nv: &PackageNv,
dist: &NpmPackageVersionDistInfo,
) -> Result<(), AnyError> {
let cache_item = {
let mut mem_cache = self.memory_cache.lock();
if let Some(cache_item) = mem_cache.get(package_nv) {
cache_item.clone()
} else {
let value_creator = MultiRuntimeAsyncValueCreator::new({
let tarball_cache = self.clone();
let package_nv = package_nv.clone();
let dist = dist.clone();
Box::new(move || {
tarball_cache.create_setup_future(package_nv.clone(), dist.clone())
})
});
let cache_item = MemoryCacheItem::Pending(Arc::new(value_creator));
mem_cache.insert(package_nv.clone(), cache_item.clone());
cache_item
}
};
match cache_item {
MemoryCacheItem::Cached => Ok(()),
MemoryCacheItem::Errored(err) => Err(anyhow!("{}", err)),
MemoryCacheItem::Pending(creator) => {
let result = creator.get().await;
match result {
Ok(_) => {
*self.memory_cache.lock().get_mut(package_nv).unwrap() =
MemoryCacheItem::Cached;
Ok(())
}
Err(err) => {
let result_err = anyhow!("{}", err);
*self.memory_cache.lock().get_mut(package_nv).unwrap() =
MemoryCacheItem::Errored(err);
Err(result_err)
}
}
}
}
}
fn create_setup_future(
self: &Arc<Self>,
package_nv: PackageNv,
dist: NpmPackageVersionDistInfo,
) -> LoadFuture {
let tarball_cache = self.clone();
async move {
let registry_url = tarball_cache.npmrc.get_registry_url(&package_nv.name);
let package_folder =
tarball_cache.cache.package_folder_for_nv_and_url(&package_nv, registry_url);
let should_use_cache = tarball_cache.cache.should_use_cache_for_package(&package_nv);
let package_folder_exists = tarball_cache.fs.exists_sync(&package_folder);
if should_use_cache && package_folder_exists {
return Ok(());
} else if tarball_cache.cache.cache_setting() == &CacheSetting::Only {
return Err(custom_error(
"NotCached",
format!(
"An npm specifier not found in cache: \"{}\", --cached-only is specified.",
&package_nv.name
)
)
);
}
if dist.tarball.is_empty() {
bail!("Tarball URL was empty.");
}
// IMPORTANT: npm registries may specify tarball URLs at different URLS than the
// registry, so we MUST get the auth for the tarball URL and not the registry URL.
let tarball_uri = Url::parse(&dist.tarball)?;
let maybe_registry_config =
tarball_cache.npmrc.tarball_config(&tarball_uri);
let maybe_auth_header = maybe_registry_config.and_then(|c| maybe_auth_header_for_npm_registry(c).ok()?);
let guard = tarball_cache.progress_bar.update(&dist.tarball);
let result = tarball_cache.http_client_provider
.get_or_create()?
.download_with_progress_and_retries(tarball_uri, maybe_auth_header, &guard)
.await;
let maybe_bytes = match result {
Ok(maybe_bytes) => maybe_bytes,
Err(DownloadError::BadResponse(err)) => {
if err.status_code == StatusCode::UNAUTHORIZED
&& maybe_registry_config.is_none()
&& tarball_cache.npmrc.get_registry_config(&package_nv.name).auth_token.is_some()
{
bail!(
concat!(
"No auth for tarball URI, but present for scoped registry.\n\n",
"Tarball URI: {}\n",
"Scope URI: {}\n\n",
"More info here: https://github.com/npm/cli/wiki/%22No-auth-for-URI,-but-auth-present-for-scoped-registry%22"
),
dist.tarball,
registry_url,
)
}
return Err(err.into())
},
Err(err) => return Err(err.into()),
};
match maybe_bytes {
Some(bytes) => {
let extraction_mode = if should_use_cache || !package_folder_exists {
TarballExtractionMode::SiblingTempDir
} else {
// The user ran with `--reload`, so overwrite the package instead of
// deleting it since the package might get corrupted if a user kills
// their deno process while it's deleting a package directory
//
// We can't rename this folder and delete it because the folder
// may be in use by another process or may now contain hardlinks,
// which will cause windows to throw an "AccessDenied" error when
// renaming. So we settle for overwriting.
TarballExtractionMode::Overwrite
};
let dist = dist.clone();
let package_nv = package_nv.clone();
deno_core::unsync::spawn_blocking(move || {
verify_and_extract_tarball(
&package_nv,
&bytes,
&dist,
&package_folder,
extraction_mode,
)
})
.await?
}
None => {
bail!("Could not find npm package tarball at: {}", dist.tarball);
}
}
}
.map(|r| r.map_err(Arc::new))
.boxed_local()
}
}

324
cli/npm/managed/cache/tarball_extract.rs vendored
View file

@ -1,324 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::fs;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_npm::registry::NpmPackageVersionDistInfo;
use deno_npm::registry::NpmPackageVersionDistInfoIntegrity;
use deno_semver::package::PackageNv;
use flate2::read::GzDecoder;
use tar::Archive;
use tar::EntryType;
use crate::util::path::get_atomic_dir_path;
#[derive(Debug, Copy, Clone)]
pub enum TarballExtractionMode {
/// Overwrites the destination directory without deleting any files.
Overwrite,
/// Creates and writes to a sibling temporary directory. When done, moves
/// it to the final destination.
///
/// This is more robust than `Overwrite` as it better handles multiple
/// processes writing to the directory at the same time.
SiblingTempDir,
}
pub fn verify_and_extract_tarball(
package_nv: &PackageNv,
data: &[u8],
dist_info: &NpmPackageVersionDistInfo,
output_folder: &Path,
extraction_mode: TarballExtractionMode,
) -> Result<(), AnyError> {
verify_tarball_integrity(package_nv, data, &dist_info.integrity())?;
match extraction_mode {
TarballExtractionMode::Overwrite => extract_tarball(data, output_folder),
TarballExtractionMode::SiblingTempDir => {
let temp_dir = get_atomic_dir_path(output_folder);
extract_tarball(data, &temp_dir)?;
rename_with_retries(&temp_dir, output_folder)
.map_err(AnyError::from)
.context("Failed moving extracted tarball to final destination.")
}
}
}
fn rename_with_retries(
temp_dir: &Path,
output_folder: &Path,
) -> Result<(), std::io::Error> {
fn already_exists(err: &std::io::Error, output_folder: &Path) -> bool {
// Windows will do an "Access is denied" error
err.kind() == ErrorKind::AlreadyExists || output_folder.exists()
}
let mut count = 0;
// renaming might be flaky if a lot of processes are trying
// to do this, so retry a few times
loop {
match fs::rename(temp_dir, output_folder) {
Ok(_) => return Ok(()),
Err(err) if already_exists(&err, output_folder) => {
// another process copied here, just cleanup
let _ = fs::remove_dir_all(temp_dir);
return Ok(());
}
Err(err) => {
count += 1;
if count > 5 {
// too many retries, cleanup and return the error
let _ = fs::remove_dir_all(temp_dir);
return Err(err);
}
// wait a bit before retrying... this should be very rare or only
// in error cases, so ok to sleep a bit
let sleep_ms = std::cmp::min(100, 20 * count);
std::thread::sleep(std::time::Duration::from_millis(sleep_ms));
}
}
}
}
fn verify_tarball_integrity(
package: &PackageNv,
data: &[u8],
npm_integrity: &NpmPackageVersionDistInfoIntegrity,
) -> Result<(), AnyError> {
use ring::digest::Context;
let (tarball_checksum, expected_checksum) = match npm_integrity {
NpmPackageVersionDistInfoIntegrity::Integrity {
algorithm,
base64_hash,
} => {
let algo = match *algorithm {
"sha512" => &ring::digest::SHA512,
"sha1" => &ring::digest::SHA1_FOR_LEGACY_USE_ONLY,
hash_kind => bail!(
"Not implemented hash function for {}: {}",
package,
hash_kind
),
};
let mut hash_ctx = Context::new(algo);
hash_ctx.update(data);
let digest = hash_ctx.finish();
let tarball_checksum = BASE64_STANDARD.encode(digest.as_ref());
(tarball_checksum, base64_hash)
}
NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(hex) => {
let mut hash_ctx = Context::new(&ring::digest::SHA1_FOR_LEGACY_USE_ONLY);
hash_ctx.update(data);
let digest = hash_ctx.finish();
let tarball_checksum = faster_hex::hex_string(digest.as_ref());
(tarball_checksum, hex)
}
NpmPackageVersionDistInfoIntegrity::UnknownIntegrity(integrity) => {
bail!(
"Not implemented integrity kind for {}: {}",
package,
integrity
)
}
};
if tarball_checksum != *expected_checksum {
bail!(
"Tarball checksum did not match what was provided by npm registry for {}.\n\nExpected: {}\nActual: {}",
package,
expected_checksum,
tarball_checksum,
)
}
Ok(())
}
fn extract_tarball(data: &[u8], output_folder: &Path) -> Result<(), AnyError> {
fs::create_dir_all(output_folder)?;
let output_folder = fs::canonicalize(output_folder)?;
let tar = GzDecoder::new(data);
let mut archive = Archive::new(tar);
archive.set_overwrite(true);
archive.set_preserve_permissions(true);
let mut created_dirs = HashSet::new();
for entry in archive.entries()? {
let mut entry = entry?;
let path = entry.path()?;
let entry_type = entry.header().entry_type();
// Some package tarballs contain "pax_global_header", these entries
// should be skipped.
if entry_type == EntryType::XGlobalHeader {
continue;
}
// skip the first component which will be either "package" or the name of the package
let relative_path = path.components().skip(1).collect::<PathBuf>();
let absolute_path = output_folder.join(relative_path);
let dir_path = if entry_type == EntryType::Directory {
absolute_path.as_path()
} else {
absolute_path.parent().unwrap()
};
if created_dirs.insert(dir_path.to_path_buf()) {
fs::create_dir_all(dir_path)?;
let canonicalized_dir = fs::canonicalize(dir_path)?;
if !canonicalized_dir.starts_with(&output_folder) {
bail!(
"Extracted directory '{}' of npm tarball was not in output directory.",
canonicalized_dir.display()
)
}
}
let entry_type = entry.header().entry_type();
match entry_type {
EntryType::Regular => {
entry.unpack(&absolute_path)?;
}
EntryType::Symlink | EntryType::Link => {
// At the moment, npm doesn't seem to support uploading hardlinks or
// symlinks to the npm registry. If ever adding symlink or hardlink
// support, we will need to validate that the hardlink and symlink
// target are within the package directory.
log::warn!(
"Ignoring npm tarball entry type {:?} for '{}'",
entry_type,
absolute_path.display()
)
}
_ => {
// ignore
}
}
}
Ok(())
}
#[cfg(test)]
mod test {
use deno_semver::Version;
use test_util::TempDir;
use super::*;
#[test]
pub fn test_verify_tarball() {
let package = PackageNv {
name: "package".to_string(),
version: Version::parse_from_npm("1.0.0").unwrap(),
};
let actual_checksum =
"z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==";
assert_eq!(
verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::UnknownIntegrity("test")
)
.unwrap_err()
.to_string(),
"Not implemented integrity kind for package@1.0.0: test",
);
assert_eq!(
verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::Integrity {
algorithm: "notimplemented",
base64_hash: "test"
}
)
.unwrap_err()
.to_string(),
"Not implemented hash function for package@1.0.0: notimplemented",
);
assert_eq!(
verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::Integrity {
algorithm: "sha1",
base64_hash: "test"
}
)
.unwrap_err()
.to_string(),
concat!(
"Tarball checksum did not match what was provided by npm ",
"registry for package@1.0.0.\n\nExpected: test\nActual: 2jmj7l5rSw0yVb/vlWAYkK/YBwk=",
),
);
assert_eq!(
verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::Integrity {
algorithm: "sha512",
base64_hash: "test"
}
)
.unwrap_err()
.to_string(),
format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {actual_checksum}"),
);
assert!(verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::Integrity {
algorithm: "sha512",
base64_hash: actual_checksum,
},
)
.is_ok());
let actual_hex = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
assert_eq!(
verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::LegacySha1Hex("test"),
)
.unwrap_err()
.to_string(),
format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {actual_hex}"),
);
assert!(verify_tarball_integrity(
&package,
&Vec::new(),
&NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(actual_hex),
)
.is_ok());
}
#[test]
fn rename_with_retries_succeeds_exists() {
let temp_dir = TempDir::new();
let folder_1 = temp_dir.path().join("folder_1");
let folder_2 = temp_dir.path().join("folder_2");
folder_1.create_dir_all();
folder_1.join("a.txt").write("test");
folder_2.create_dir_all();
// this will not end up in the output as rename_with_retries assumes
// the folders ending up at the destination are the same
folder_2.join("b.txt").write("test2");
let dest_folder = temp_dir.path().join("dest_folder");
rename_with_retries(folder_1.as_path(), dest_folder.as_path()).unwrap();
rename_with_retries(folder_2.as_path(), dest_folder.as_path()).unwrap();
assert!(dest_folder.join("a.txt").exists());
assert!(!dest_folder.join("b.txt").exists());
}
}

73
cli/npm/managed/mod.rs
View file

@ -5,8 +5,6 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use cache::RegistryInfoDownloader;
use cache::TarballCache;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
@ -42,22 +40,23 @@ use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
use self::cache::NpmCache;
use self::registry::CliNpmRegistryApi;
use self::resolution::NpmResolution;
use self::resolvers::create_npm_fs_resolver;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmCache;
use super::CliNpmCacheEnv;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
use super::InnerCliNpmResolverRef;
use super::ResolvePkgFolderFromDenoReqError;
pub mod cache;
mod registry;
mod resolution;
mod resolvers;
@ -85,8 +84,9 @@ pub struct CliManagedNpmResolverCreateOptions {
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
let cache_env = create_cache_env(&options);
let npm_cache = create_cache(cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
@ -97,8 +97,8 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
};
create_inner(
cache_env,
options.fs,
options.http_client_provider,
options.maybe_lockfile,
npm_api,
npm_cache,
@ -118,12 +118,13 @@ pub async fn create_managed_npm_resolver_for_lsp(
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache = create_cache(&options);
let npm_api = create_api(&options, npm_cache.clone());
let npm_cache_env = create_cache_env(&options);
let npm_cache = create_cache(npm_cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options);
let snapshot = resolve_snapshot(&npm_api, options.snapshot).await?;
Ok(create_inner(
npm_cache_env,
options.fs,
options.http_client_provider,
options.maybe_lockfile,
npm_api,
npm_cache,
@ -139,11 +140,11 @@ pub async fn create_managed_npm_resolver(
#[allow(clippy::too_many_arguments)]
fn create_inner(
env: Arc<CliNpmCacheEnv>,
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
@ -157,12 +158,10 @@ fn create_inner(
snapshot,
maybe_lockfile.clone(),
));
let tarball_cache = Arc::new(TarballCache::new(
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
fs.clone(),
http_client_provider.clone(),
env,
npm_rc.clone(),
text_only_progress_bar.clone(),
));
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
@ -190,25 +189,39 @@ fn create_inner(
))
}
fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc<NpmCache> {
Arc::new(NpmCache::new(
fn create_cache_env(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCacheEnv> {
Arc::new(CliNpmCacheEnv::new(
options.fs.clone(),
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
))
}
fn create_cache(
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.cache_setting.clone(),
options.cache_setting.as_npm_cache_setting(),
env,
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
npm_cache: Arc<NpmCache>,
) -> Arc<CliNpmRegistryApi> {
Arc::new(CliNpmRegistryApi::new(
npm_cache.clone(),
Arc::new(RegistryInfoDownloader::new(
npm_cache,
options.http_client_provider.clone(),
cache.clone(),
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
env,
options.npmrc.clone(),
options.text_only_progress_bar.clone(),
)),
))
}
@ -292,10 +305,10 @@ pub struct ManagedCliNpmResolver {
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
top_level_install_flag: AtomicFlag,
@ -317,10 +330,10 @@ impl ManagedCliNpmResolver {
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

23
cli/npm/managed/registry.rs
View file

@ -14,27 +14,28 @@ use deno_core::parking_lot::Mutex;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::registry::NpmRegistryApi;
use deno_npm::registry::NpmRegistryPackageInfoLoadError;
use deno_npm_cache::NpmCacheSetting;
use crate::args::CacheSetting;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmRegistryInfoProvider;
use crate::util::sync::AtomicFlag;
use super::cache::NpmCache;
use super::cache::RegistryInfoDownloader;
// todo(#27198): Remove this and move functionality down into
// RegistryInfoProvider, which already does most of this.
#[derive(Debug)]
pub struct CliNpmRegistryApi(Option<Arc<CliNpmRegistryApiInner>>);
impl CliNpmRegistryApi {
pub fn new(
cache: Arc<NpmCache>,
registry_info_downloader: Arc<RegistryInfoDownloader>,
cache: Arc<CliNpmCache>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
) -> Self {
Self(Some(Arc::new(CliNpmRegistryApiInner {
cache,
force_reload_flag: Default::default(),
mem_cache: Default::default(),
previously_reloaded_packages: Default::default(),
registry_info_downloader,
registry_info_provider,
})))
}
@ -83,11 +84,11 @@ enum CacheItem {
#[derive(Debug)]
struct CliNpmRegistryApiInner {
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
force_reload_flag: AtomicFlag,
mem_cache: Mutex<HashMap<String, CacheItem>>,
previously_reloaded_packages: Mutex<HashSet<String>>,
registry_info_downloader: Arc<RegistryInfoDownloader>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
}
impl CliNpmRegistryApiInner {
@ -118,7 +119,7 @@ impl CliNpmRegistryApiInner {
return Ok(result);
}
}
api.registry_info_downloader
api.registry_info_provider
.load_package_info(&name)
.await
.map_err(Arc::new)
@ -159,7 +160,7 @@ impl CliNpmRegistryApiInner {
// is disabled or if we're already reloading
if matches!(
self.cache.cache_setting(),
CacheSetting::Only | CacheSetting::ReloadAll
NpmCacheSetting::Only | NpmCacheSetting::ReloadAll
) {
return false;
}

4
cli/npm/managed/resolvers/common.rs
View file

@ -24,7 +24,7 @@ use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use crate::npm::managed::cache::TarballCache;
use crate::npm::CliNpmTarballCache;
/// Part of the resolution that interacts with the file system.
#[async_trait(?Send)]
@ -140,7 +140,7 @@ impl RegistryReadPermissionChecker {
/// Caches all the packages in parallel.
pub async fn cache_packages(
packages: &[NpmResolutionPackage],
tarball_cache: &Arc<TarballCache>,
tarball_cache: &Arc<CliNpmTarballCache>,
) -> Result<(), AnyError> {
let mut futures_unordered = futures::stream::FuturesUnordered::new();
for package in packages {

12
cli/npm/managed/resolvers/global.rs
View file

@ -8,6 +8,8 @@ use std::path::PathBuf;
use std::sync::Arc;
use crate::colors;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
@ -24,8 +26,6 @@ use node_resolver::errors::ReferrerNotFoundError;
use crate::args::LifecycleScriptsConfig;
use crate::cache::FastInsecureHasher;
use super::super::cache::NpmCache;
use super::super::cache::TarballCache;
use super::super::resolution::NpmResolution;
use super::common::cache_packages;
use super::common::lifecycle_scripts::LifecycleScriptsStrategy;
@ -35,8 +35,8 @@ use super::common::RegistryReadPermissionChecker;
/// Resolves packages from the global npm cache.
#[derive(Debug)]
pub struct GlobalNpmPackageResolver {
cache: Arc<NpmCache>,
tarball_cache: Arc<TarballCache>,
cache: Arc<CliNpmCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
registry_read_permission_checker: RegistryReadPermissionChecker,
@ -45,9 +45,9 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn FileSystem>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

16
cli/npm/managed/resolvers/local.rs
View file

@ -17,6 +17,8 @@ use std::sync::Arc;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
@ -52,8 +54,6 @@ use crate::util::fs::LaxSingleProcessFsFlag;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressMessagePrompt;
use super::super::cache::NpmCache;
use super::super::cache::TarballCache;
use super::super::resolution::NpmResolution;
use super::common::bin_entries;
use super::common::NpmPackageFsResolver;
@ -63,12 +63,12 @@ use super::common::RegistryReadPermissionChecker;
/// and resolves packages from it.
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
system_info: NpmSystemInfo,
@ -79,12 +79,12 @@ pub struct LocalNpmPackageResolver {
impl LocalNpmPackageResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<NpmCache>,
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
@ -284,10 +284,10 @@ fn local_node_modules_package_contents_path(
#[allow(clippy::too_many_arguments)]
async fn sync_resolution_with_fs(
snapshot: &NpmResolutionSnapshot,
cache: &Arc<NpmCache>,
cache: &Arc<CliNpmCache>,
npm_install_deps_provider: &NpmInstallDepsProvider,
progress_bar: &ProgressBar,
tarball_cache: &Arc<TarballCache>,
tarball_cache: &Arc<CliNpmTarballCache>,
root_node_modules_dir_path: &Path,
system_info: &NpmSystemInfo,
lifecycle_scripts: &LifecycleScriptsConfig,

8
cli/npm/managed/resolvers/mod.rs
View file

@ -12,6 +12,8 @@ use deno_runtime::deno_fs::FileSystem;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::util::progress_bar::ProgressBar;
pub use self::common::NpmPackageFsResolver;
@ -19,18 +21,16 @@ pub use self::common::NpmPackageFsResolver;
use self::global::GlobalNpmPackageResolver;
use self::local::LocalNpmPackageResolver;
use super::cache::NpmCache;
use super::cache::TarballCache;
use super::resolution::NpmResolution;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<NpmCache>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,

111
cli/npm/mod.rs
View file

@ -1,33 +1,39 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod byonm;
mod common;
mod managed;
use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
use common::maybe_auth_header_for_npm_registry;
use dashmap::DashMap;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
use deno_semver::package::PackageReq;
use managed::cache::registry_info::get_package_url;
use http::HeaderName;
use http::HeaderValue;
use managed::create_managed_in_npm_pkg_checker;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmPackageFolderResolver;
use crate::file_fetcher::FileFetcher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::hard_link_dir_recursive;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::progress_bar::ProgressBar;
pub use self::byonm::CliByonmNpmResolver;
pub use self::byonm::CliByonmNpmResolverCreateOptions;
@ -36,6 +42,99 @@ pub use self::managed::CliManagedNpmResolverCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub type CliNpmTarballCache = deno_npm_cache::TarballCache<CliNpmCacheEnv>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliNpmCacheEnv>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheEnv>;
#[derive(Debug)]
pub struct CliNpmCacheEnv {
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
}
impl CliNpmCacheEnv {
pub fn new(
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
) -> Self {
Self {
fs,
http_client_provider,
progress_bar,
}
}
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
fn exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
// todo(dsherret): use self.fs here instead
hard_link_dir_recursive(from, to)
}
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.fs.as_ref(),
write_mode: crate::cache::CACHE_PERM,
},
file_path,
data,
)
}
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,
maybe_auth_header: Option<(HeaderName, HeaderValue)>,
) -> Result<Option<Vec<u8>>, deno_npm_cache::DownloadError> {
let guard = self.progress_bar.update(url.as_str());
let client = self.http_client_provider.get_or_create().map_err(|err| {
deno_npm_cache::DownloadError {
status_code: None,
error: err,
}
})?;
client
.download_with_progress_and_retries(url, maybe_auth_header, &guard)
.await
.map_err(|err| {
use crate::http_util::DownloadError::*;
let status_code = match &err {
Fetch { .. }
| UrlParse { .. }
| HttpParse { .. }
| Json { .. }
| ToStr { .. }
| NoRedirectHeader { .. }
| TooManyRedirects => None,
BadResponse(bad_response_error) => {
Some(bad_response_error.status_code)
}
};
deno_npm_cache::DownloadError {
status_code,
error: err.into(),
}
})
}
}
pub enum CliNpmResolverCreateOptions {
Managed(CliManagedNpmResolverCreateOptions),
Byonm(CliByonmNpmResolverCreateOptions),
@ -179,13 +278,15 @@ impl NpmFetchResolver {
if let Some(info) = self.info_by_name.get(name) {
return info.value().clone();
}
// todo(#27198): use RegistryInfoProvider instead
let fetch_package_info = || async {
let info_url = get_package_url(&self.npmrc, name);
let info_url = deno_npm_cache::get_package_url(&self.npmrc, name);
let file_fetcher = self.file_fetcher.clone();
let registry_config = self.npmrc.get_registry_config(name);
// TODO(bartlomieju): this should error out, not use `.ok()`.
let maybe_auth_header =
maybe_auth_header_for_npm_registry(registry_config).ok()?;
deno_npm_cache::maybe_auth_header_for_npm_registry(registry_config)
.ok()?;
// spawn due to the lsp's `Send` requirement
let file = deno_core::unsync::spawn(async move {
file_fetcher