language-servers/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-07-24 13:44:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

refactor: move NpmCacheDir to deno_cache_dir (#25916)

Browse source 
Part of the ongoing work to move more of Deno's resolution out of the
CLI crate (for use in Wasm and other things)

Includes:

* https://github.com/denoland/deno_cache_dir/pull/60
This commit is contained in:
David Sherret 2024-09-28 08:50:16 -04:00 • committed by GitHub
parent fc739dc5eb
commit 1bb47805d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 267 additions and 411 deletions
Download patch file Download diff file Expand all files Collapse all files

295
cli/npm/cache_dir.rs
View file

@ -1,295 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use crate::util::fs::canonicalize_path;
use crate::util::path::root_url_to_safe_local_dirname;
/// The global cache directory of npm packages.
#[derive(Clone, Debug)]
pub struct NpmCacheDir {
root_dir: PathBuf,
// cached url representation of the root directory
root_dir_url: Url,
// A list of all registry that were discovered via `.npmrc` files
// turned into a safe directory names.
known_registries_dirnames: Vec<String>,
}
impl NpmCacheDir {
pub fn new(root_dir: PathBuf, known_registries_urls: Vec<Url>) -> Self {
fn try_get_canonicalized_root_dir(
root_dir: &Path,
) -> Result<PathBuf, AnyError> {
if !root_dir.exists() {
std::fs::create_dir_all(root_dir)
.with_context(|| format!("Error creating {}", root_dir.display()))?;
}
Ok(canonicalize_path(root_dir)?)
}
// this may fail on readonly file systems, so just ignore if so
let root_dir =
try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir);
let root_dir_url = Url::from_directory_path(&root_dir).unwrap();
let known_registries_dirnames: Vec<_> = known_registries_urls
.into_iter()
.map(|url| {
root_url_to_safe_local_dirname(&url)
.to_string_lossy()
.replace('\\', "/")
})
.collect();
Self {
root_dir,
root_dir_url,
known_registries_dirnames,
}
}
pub fn root_dir(&self) -> &Path {
&self.root_dir
}
pub fn root_dir_url(&self) -> &Url {
&self.root_dir_url
}
pub fn package_folder_for_id(
&self,
folder_id: &NpmPackageCacheFolderId,
registry_url: &Url,
) -> PathBuf {
if folder_id.copy_index == 0 {
self.package_folder_for_nv(&folder_id.nv, registry_url)
} else {
self
.package_name_folder(&folder_id.nv.name, registry_url)
.join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index))
}
}
pub fn package_folder_for_nv(
&self,
package: &PackageNv,
registry_url: &Url,
) -> PathBuf {
self
.package_name_folder(&package.name, registry_url)
.join(package.version.to_string())
}
pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf {
let mut dir = self.registry_folder(registry_url);
if name.to_lowercase() != name {
let encoded_name = mixed_case_package_name_encode(name);
// Using the encoded directory may have a collision with an actual package name
// so prefix it with an underscore since npm packages can't start with that
dir.join(format!("_{encoded_name}"))
} else {
// ensure backslashes are used on windows
for part in name.split('/') {
dir = dir.join(part);
}
dir
}
}
fn registry_folder(&self, registry_url: &Url) -> PathBuf {
self
.root_dir
.join(root_url_to_safe_local_dirname(registry_url))
}
pub fn resolve_package_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Option<NpmPackageCacheFolderId> {
let mut maybe_relative_url = None;
// Iterate through known registries and try to get a match.
for registry_dirname in &self.known_registries_dirnames {
let registry_root_dir = self
.root_dir_url
.join(&format!("{}/", registry_dirname))
// this not succeeding indicates a fatal issue, so unwrap
.unwrap();
let Some(relative_url) = registry_root_dir.make_relative(specifier)
else {
continue;
};
if relative_url.starts_with("../") {
continue;
}
maybe_relative_url = Some(relative_url);
break;
}
let mut relative_url = maybe_relative_url?;
// base32 decode the url if it starts with an underscore
// * Ex. _{base32(package_name)}/
if let Some(end_url) = relative_url.strip_prefix('_') {
let mut parts = end_url
.split('/')
.map(ToOwned::to_owned)
.collect::<Vec<_>>();
match mixed_case_package_name_decode(&parts[0]) {
Some(part) => {
parts[0] = part;
}
None => return None,
}
relative_url = parts.join("/");
}
// examples:
// * chalk/5.0.1/
// * @types/chalk/5.0.1/
// * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps
let is_scoped_package = relative_url.starts_with('@');
let mut parts = relative_url
.split('/')
.enumerate()
.take(if is_scoped_package { 3 } else { 2 })
.map(|(_, part)| part)
.collect::<Vec<_>>();
if parts.len() < 2 {
return None;
}
let version_part = parts.pop().unwrap();
let name = parts.join("/");
let (version, copy_index) =
if let Some((version, copy_count)) = version_part.split_once('_') {
(version, copy_count.parse::<u8>().ok()?)
} else {
(version_part, 0)
};
Some(NpmPackageCacheFolderId {
nv: PackageNv {
name,
version: Version::parse_from_npm(version).ok()?,
},
copy_index,
})
}
pub fn get_cache_location(&self) -> PathBuf {
self.root_dir.clone()
}
}
pub fn mixed_case_package_name_encode(name: &str) -> String {
// use base32 encoding because it's reversible and the character set
// only includes the characters within 0-9 and A-Z so it can be lower cased
base32::encode(
base32::Alphabet::Rfc4648Lower { padding: false },
name.as_bytes(),
)
.to_lowercase()
}
pub fn mixed_case_package_name_decode(name: &str) -> Option<String> {
base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name)
.and_then(|b| String::from_utf8(b).ok())
}
#[cfg(test)]
mod test {
use deno_core::url::Url;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use super::NpmCacheDir;
use crate::npm::cache_dir::NpmPackageCacheFolderId;
#[test]
fn should_get_package_folder() {
let deno_dir = crate::cache::DenoDir::new(None).unwrap();
let root_dir = deno_dir.npm_folder_path();
let registry_url = Url::parse("https://registry.npmjs.org/").unwrap();
let cache = NpmCacheDir::new(root_dir.clone(), vec![registry_url.clone()]);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "json".to_string(),
version: Version::parse_from_npm("1.2.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("json")
.join("1.2.5"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "json".to_string(),
version: Version::parse_from_npm("1.2.5").unwrap(),
},
copy_index: 1,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("json")
.join("1.2.5_1"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "JSON".to_string(),
version: Version::parse_from_npm("2.1.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("_jjju6tq")
.join("2.1.5"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "@types/JSON".to_string(),
version: Version::parse_from_npm("2.1.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("_ib2hs4dfomxuuu2pjy")
.join("2.1.5"),
);
}
}

44
cli/npm/managed/cache/mod.rs vendored
View file

@ -8,6 +8,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
@ -18,10 +19,10 @@ use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use crate::args::CacheSetting;
use crate::cache::CACHE_PERM;
use crate::npm::NpmCacheDir;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::hard_link_dir_recursive;
@ -87,9 +88,12 @@ impl NpmCache {
) -> Result<(), AnyError> {
let registry_url = self.npmrc.get_registry_url(&folder_id.nv.name);
assert_ne!(folder_id.copy_index, 0);
let package_folder = self
.cache_dir
.package_folder_for_id(folder_id, registry_url);
let package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
folder_id.copy_index,
registry_url,
);
if package_folder.exists()
// if this file exists, then the package didn't successfully initialize
@ -100,9 +104,12 @@ impl NpmCache {
return Ok(());
}
let original_package_folder = self
.cache_dir
.package_folder_for_nv(&folder_id.nv, registry_url);
let original_package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
0, // original copy index
registry_url,
);
// it seems Windows does an "AccessDenied" error when moving a
// directory with hard links, so that's why this solution is done
@ -114,7 +121,12 @@ impl NpmCache {
pub fn package_folder_for_id(&self, id: &NpmPackageCacheFolderId) -> PathBuf {
let registry_url = self.npmrc.get_registry_url(&id.nv.name);
self.cache_dir.package_folder_for_id(id, registry_url)
self.cache_dir.package_folder_for_id(
&id.nv.name,
&id.nv.version.to_string(),
id.copy_index,
registry_url,
)
}
pub fn package_folder_for_nv(&self, package: &PackageNv) -> PathBuf {
@ -127,7 +139,12 @@ impl NpmCache {
package: &PackageNv,
registry_url: &Url,
) -> PathBuf {
self.cache_dir.package_folder_for_nv(package, registry_url)
self.cache_dir.package_folder_for_id(
&package.name,
&package.version.to_string(),
0, // original copy_index
registry_url,
)
}
pub fn package_name_folder(&self, name: &str) -> PathBuf {
@ -146,6 +163,15 @@ impl NpmCache {
self
.cache_dir
.resolve_package_folder_id_from_specifier(specifier)
.and_then(|cache_id| {
Some(NpmPackageCacheFolderId {
nv: PackageNv {
name: cache_id.name,
version: Version::parse_from_npm(&cache_id.version).ok()?,
},
copy_index: cache_id.copy_index,
})
})
}
pub fn load_package_info(

4
cli/npm/managed/mod.rs
View file

@ -7,6 +7,7 @@ use std::sync::Arc;
use cache::RegistryInfoDownloader;
use cache::TarballCache;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::serde_json;
@ -35,6 +36,7 @@ use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
@ -50,7 +52,6 @@ use self::resolvers::NpmPackageFsResolver;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
use super::NpmCacheDir;
mod cache;
mod registry;
@ -188,6 +189,7 @@ fn create_inner(
fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc<NpmCache> {
Arc::new(NpmCache::new(
NpmCacheDir::new(
&DenoCacheEnvFsAdapter(options.fs.as_ref()),
options.npm_global_cache_dir.clone(),
options.npmrc.get_all_known_registries_urls(),
),

4
cli/npm/managed/resolvers/local.rs
View file

@ -19,6 +19,8 @@ use crate::args::LifecycleScriptsConfig;
use crate::colors;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
use deno_cache_dir::npm::mixed_case_package_name_encode;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::stream::FuturesUnordered;
@ -42,8 +44,6 @@ use serde::Serialize;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::npm::cache_dir::mixed_case_package_name_decode;
use crate::npm::cache_dir::mixed_case_package_name_encode;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::fs::clone_dir_recursive;

2
cli/npm/mod.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod byonm;
mod cache_dir;
mod common;
mod managed;
@ -24,7 +23,6 @@ use crate::file_fetcher::FileFetcher;
pub use self::byonm::ByonmCliNpmResolver;
pub use self::byonm::CliNpmResolverByonmCreateOptions;
pub use self::cache_dir::NpmCacheDir;
pub use self::managed::CliNpmResolverManagedCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;