diff --git a/cli/factory.rs b/cli/factory.rs index 5d18303987..61816af00e 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -18,7 +18,6 @@ use deno_error::JsErrorBox; use deno_lib::args::CaData; use deno_lib::args::get_root_cert_store; use deno_lib::args::npm_process_state; -use deno_lib::loader::NpmModuleLoader; use deno_lib::npm::NpmRegistryReadPermissionChecker; use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; use deno_lib::npm::create_npm_process_state_provider; @@ -915,7 +914,6 @@ impl CliFactory { let in_npm_pkg_checker = self.in_npm_pkg_checker()?; let workspace_factory = self.workspace_factory()?; let resolver_factory = self.resolver_factory()?; - let node_code_translator = resolver_factory.node_code_translator()?; let cjs_tracker = self.cjs_tracker()?.clone(); let npm_registry_permission_checker = { let mode = if resolver_factory.use_byonm()? { @@ -949,11 +947,7 @@ impl CliFactory { in_npm_pkg_checker.clone(), self.main_module_graph_container().await?.clone(), self.module_load_preparer().await?.clone(), - NpmModuleLoader::new( - self.cjs_tracker()?.clone(), - node_code_translator.clone(), - self.sys(), - ), + resolver_factory.npm_module_loader()?.clone(), npm_registry_permission_checker, cli_npm_resolver.clone(), resolver_factory.parsed_source_cache().clone(), diff --git a/cli/lib/loader.rs b/cli/lib/loader.rs index ce04d06185..58be3de7a7 100644 --- a/cli/lib/loader.rs +++ b/cli/lib/loader.rs @@ -1,228 +1,75 @@ // Copyright 2018-2025 the Deno authors. MIT license. use std::borrow::Cow; -use std::path::PathBuf; -use std::sync::Arc; use deno_media_type::MediaType; -use deno_resolver::cjs::CjsTracker; -use deno_resolver::npm::DenoInNpmPackageChecker; +use deno_resolver::loader::LoadedModuleSource; +use deno_runtime::deno_core::FastString; use deno_runtime::deno_core::ModuleSourceCode; use deno_runtime::deno_core::ModuleType; -use node_resolver::InNpmPackageChecker; -use node_resolver::IsBuiltInNodeModuleChecker; -use node_resolver::NpmPackageFolderResolver; -use node_resolver::analyze::CjsCodeAnalyzer; -use node_resolver::analyze::NodeCodeTranslator; -use thiserror::Error; -use url::Url; +use deno_runtime::deno_core::RequestedModuleType; -use crate::sys::DenoLibSys; -use crate::util::text_encoding::from_utf8_lossy_cow; - -pub struct ModuleCodeStringSource { - pub code: ModuleSourceCode, - pub found_url: Url, - pub module_type: ModuleType, -} - -#[derive(Debug, Error, deno_error::JsError)] -#[class(type)] -#[error("[{}]: Stripping types is currently unsupported for files under node_modules, for \"{}\"", self.code(), specifier)] -pub struct StrippingTypesNodeModulesError { - pub specifier: Url, -} - -impl StrippingTypesNodeModulesError { - pub fn code(&self) -> &'static str { - "ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING" - } -} - -#[derive(Debug, Error, deno_error::JsError)] -pub enum NpmModuleLoadError { - #[class(inherit)] - #[error(transparent)] - UrlToFilePath(#[from] deno_path_util::UrlToFilePathError), - #[class(inherit)] - #[error(transparent)] - StrippingTypesNodeModules(#[from] StrippingTypesNodeModulesError), - #[class(inherit)] - #[error(transparent)] - ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), - #[class(inherit)] - #[error(transparent)] - TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError), - #[class(inherit)] - #[error("Unable to load {}{}", file_path.display(), maybe_referrer.as_ref().map(|r| format!(" imported from {}", r)).unwrap_or_default())] - UnableToLoad { - file_path: PathBuf, - maybe_referrer: Option, - #[source] - #[inherit] - source: std::io::Error, - }, - #[class(inherit)] - #[error( - "{}", - format_dir_import_message(file_path, maybe_referrer, suggestion) - )] - DirImport { - file_path: PathBuf, - maybe_referrer: Option, - suggestion: Option<&'static str>, - #[source] - #[inherit] - source: std::io::Error, - }, -} - -fn format_dir_import_message( - file_path: &std::path::Path, - maybe_referrer: &Option, - suggestion: &Option<&'static str>, -) -> String { - // directory imports are not allowed when importing from an - // ES module, so provide the user with a helpful error message - let dir_path = file_path; - let mut msg = "Directory import ".to_string(); - msg.push_str(&dir_path.to_string_lossy()); - if let Some(referrer) = maybe_referrer { - msg.push_str(" is not supported resolving import from "); - msg.push_str(referrer.as_str()); - if let Some(entrypoint_name) = suggestion { - msg.push_str("\nDid you mean to import "); - msg.push_str(entrypoint_name); - msg.push_str(" within the directory?"); - } - } - msg -} - -#[derive(Clone)] -pub struct NpmModuleLoader< - TCjsCodeAnalyzer: CjsCodeAnalyzer, - TInNpmPackageChecker: InNpmPackageChecker, - TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, - TNpmPackageFolderResolver: NpmPackageFolderResolver, - TSys: DenoLibSys, -> { - cjs_tracker: Arc>, - sys: TSys, - node_code_translator: Arc< - NodeCodeTranslator< - TCjsCodeAnalyzer, - TInNpmPackageChecker, - TIsBuiltInNodeModuleChecker, - TNpmPackageFolderResolver, - TSys, - >, - >, -} - -impl< - TCjsCodeAnalyzer: CjsCodeAnalyzer, - TInNpmPackageChecker: InNpmPackageChecker, - TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, - TNpmPackageFolderResolver: NpmPackageFolderResolver, - TSys: DenoLibSys, -> - NpmModuleLoader< - TCjsCodeAnalyzer, - TInNpmPackageChecker, - TIsBuiltInNodeModuleChecker, - TNpmPackageFolderResolver, - TSys, - > -{ - pub fn new( - cjs_tracker: Arc>, - node_code_translator: Arc< - NodeCodeTranslator< - TCjsCodeAnalyzer, - TInNpmPackageChecker, - TIsBuiltInNodeModuleChecker, - TNpmPackageFolderResolver, - TSys, - >, - >, - sys: TSys, - ) -> Self { - Self { - cjs_tracker, - node_code_translator, - sys, - } - } - - pub async fn load( - &self, - specifier: &Url, - maybe_referrer: Option<&Url>, - ) -> Result { - let file_path = deno_path_util::url_to_file_path(specifier)?; - let code = self.sys.fs_read(&file_path).map_err(|source| { - if self.sys.fs_is_dir_no_err(&file_path) { - let suggestion = ["index.mjs", "index.js", "index.cjs"] - .into_iter() - .find(|e| self.sys.fs_is_file_no_err(file_path.join(e))); - NpmModuleLoadError::DirImport { - file_path, - maybe_referrer: maybe_referrer.cloned(), - suggestion, - source, - } - } else { - NpmModuleLoadError::UnableToLoad { - file_path, - maybe_referrer: maybe_referrer.cloned(), - source, - } +pub fn module_type_from_media_and_requested_type( + media_type: MediaType, + requested_module_type: &RequestedModuleType, +) -> ModuleType { + match requested_module_type { + RequestedModuleType::Json => ModuleType::Json, + RequestedModuleType::Text => ModuleType::Text, + RequestedModuleType::Bytes => ModuleType::Bytes, + RequestedModuleType::None | RequestedModuleType::Other(_) => { + match media_type { + MediaType::Json => ModuleType::Json, + MediaType::Wasm => ModuleType::Wasm, + _ => ModuleType::JavaScript, } - })?; - - let media_type = MediaType::from_specifier(specifier); - if media_type.is_emittable() { - return Err(NpmModuleLoadError::StrippingTypesNodeModules( - StrippingTypesNodeModulesError { - specifier: specifier.clone(), - }, - )); } - - let code = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { - // translate cjs to esm if it's cjs and inject node globals - let code = from_utf8_lossy_cow(code); - ModuleSourceCode::String( - self - .node_code_translator - .translate_cjs_to_esm(specifier, Some(code)) - .await? - .into_owned() - .into(), - ) - } else { - // esm and json code is untouched - ModuleSourceCode::Bytes(match code { - Cow::Owned(bytes) => bytes.into_boxed_slice().into(), - Cow::Borrowed(bytes) => bytes.into(), - }) - }; - - Ok(ModuleCodeStringSource { - code, - found_url: specifier.clone(), - module_type: module_type_from_media_type(MediaType::from_specifier( - specifier, - )), - }) } } -pub fn module_type_from_media_type(media_type: MediaType) -> ModuleType { - match media_type { - MediaType::Json => ModuleType::Json, - MediaType::Wasm => ModuleType::Wasm, - _ => ModuleType::JavaScript, +pub fn loaded_module_source_to_module_source_code( + loaded_module_source: LoadedModuleSource, +) -> ModuleSourceCode { + match loaded_module_source { + LoadedModuleSource::ArcStr(text) => ModuleSourceCode::String(text.into()), + LoadedModuleSource::ArcBytes(bytes) => { + ModuleSourceCode::Bytes(bytes.into()) + } + LoadedModuleSource::String(text) => match text { + Cow::Borrowed(static_text) => { + ModuleSourceCode::String(FastString::from_static(static_text)) + } + Cow::Owned(text) => ModuleSourceCode::String(text.into()), + }, + LoadedModuleSource::Bytes(bytes) => match bytes { + Cow::Borrowed(static_bytes) => { + ModuleSourceCode::Bytes(static_bytes.into()) + } + Cow::Owned(bytes) => { + ModuleSourceCode::Bytes(bytes.into_boxed_slice().into()) + } + }, + } +} + +pub fn as_deno_resolver_requested_module_type( + value: &RequestedModuleType, +) -> deno_resolver::loader::RequestedModuleType<'_> { + match value { + RequestedModuleType::None => { + deno_resolver::loader::RequestedModuleType::None + } + RequestedModuleType::Json => { + deno_resolver::loader::RequestedModuleType::Json + } + RequestedModuleType::Text => { + deno_resolver::loader::RequestedModuleType::Text + } + RequestedModuleType::Bytes => { + deno_resolver::loader::RequestedModuleType::Bytes + } + RequestedModuleType::Other(text) => { + deno_resolver::loader::RequestedModuleType::Other(text) + } } } diff --git a/cli/lib/util/text_encoding.rs b/cli/lib/util/text_encoding.rs index 480c9776e7..ff0f862b23 100644 --- a/cli/lib/util/text_encoding.rs +++ b/cli/lib/util/text_encoding.rs @@ -7,6 +7,7 @@ pub fn is_valid_utf8(bytes: &[u8]) -> bool { matches!(String::from_utf8_lossy(bytes), Cow::Borrowed(_)) } +// todo(https://github.com/rust-lang/rust/issues/129436): remove once stabilized #[inline(always)] pub fn from_utf8_lossy_owned(bytes: Vec) -> String { match String::from_utf8_lossy(&bytes) { diff --git a/cli/module_loader.rs b/cli/module_loader.rs index d3e4b5313f..b6d2407060 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -43,10 +43,9 @@ use deno_error::JsErrorBox; use deno_graph::GraphKind; use deno_graph::ModuleGraph; use deno_graph::WalkOptions; -use deno_lib::loader::ModuleCodeStringSource; -use deno_lib::loader::NpmModuleLoadError; -use deno_lib::loader::StrippingTypesNodeModulesError; -use deno_lib::loader::module_type_from_media_type; +use deno_lib::loader::as_deno_resolver_requested_module_type; +use deno_lib::loader::loaded_module_source_to_module_source_code; +use deno_lib::loader::module_type_from_media_and_requested_type; use deno_lib::npm::NpmRegistryReadPermissionChecker; use deno_lib::util::hash::FastInsecureHasher; use deno_lib::worker::CreateModuleLoaderResult; @@ -57,8 +56,10 @@ use deno_resolver::file_fetcher::FetchPermissionsOptionRef; use deno_resolver::graph::ResolveWithGraphErrorKind; use deno_resolver::graph::ResolveWithGraphOptions; use deno_resolver::loader::LoadPreparedModuleError; -use deno_resolver::loader::PreparedModuleOrAsset; -use deno_resolver::loader::PreparedModuleSource; +use deno_resolver::loader::LoadedModule; +use deno_resolver::loader::LoadedModuleOrAsset; +use deno_resolver::loader::NpmModuleLoadError; +use deno_resolver::loader::StrippingTypesNodeModulesError; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::npm::ResolveNpmReqRefError; use deno_runtime::code_cache; @@ -69,7 +70,6 @@ use deno_runtime::deno_permissions::CheckSpecifierKind; use deno_runtime::deno_permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; use eszip::EszipV2; -use node_resolver::DenoIsBuiltInNodeModuleChecker; use node_resolver::InNpmPackageChecker; use node_resolver::NodeResolutionKind; use node_resolver::ResolutionMode; @@ -92,7 +92,6 @@ use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::BuildGraphRequest; use crate::graph_util::BuildGraphWithNpmOptions; use crate::graph_util::ModuleGraphBuilder; -use crate::node::CliCjsCodeAnalyzer; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::resolver::CliResolver; @@ -104,13 +103,8 @@ use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; -pub type CliNpmModuleLoader = deno_lib::loader::NpmModuleLoader< - CliCjsCodeAnalyzer, - DenoInNpmPackageChecker, - DenoIsBuiltInNodeModuleChecker, - CliNpmResolver, - CliSys, ->; +pub type CliNpmModuleLoader = + deno_resolver::loader::DenoNpmModuleLoader; pub type CliEmitter = deno_resolver::emit::Emitter; pub type CliPreparedModuleLoader = @@ -336,7 +330,7 @@ struct SharedCliModuleLoaderState { in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, module_load_preparer: Arc, - npm_module_loader: CliNpmModuleLoader, + npm_module_loader: Arc, npm_registry_permission_checker: Arc>, npm_resolver: CliNpmResolver, @@ -398,7 +392,7 @@ impl CliModuleLoaderFactory { in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, module_load_preparer: Arc, - npm_module_loader: CliNpmModuleLoader, + npm_module_loader: Arc, npm_registry_permission_checker: Arc< NpmRegistryReadPermissionChecker, >, @@ -530,6 +524,12 @@ impl CliModuleLoaderFactory { } } +struct ModuleCodeStringSource { + pub code: ModuleSourceCode, + pub found_url: ModuleSpecifier, + pub module_type: ModuleType, +} + #[derive(Debug, thiserror::Error, deno_error::JsError)] #[class(generic)] #[error("Loading unprepared module: {}{}", .specifier, .maybe_referrer.as_ref().map(|r| format!(", imported from: {}", r)).unwrap_or_default())] @@ -712,63 +712,28 @@ impl maybe_referrer: Option<&ModuleSpecifier>, requested_module_type: &RequestedModuleType, ) -> Result { - fn as_deno_resolver_requested_module_type( - value: &RequestedModuleType, - ) -> deno_resolver::loader::RequestedModuleType<'_> { - match value { - RequestedModuleType::None => { - deno_resolver::loader::RequestedModuleType::None - } - RequestedModuleType::Json => { - deno_resolver::loader::RequestedModuleType::Json - } - RequestedModuleType::Text => { - deno_resolver::loader::RequestedModuleType::Text - } - RequestedModuleType::Bytes => { - deno_resolver::loader::RequestedModuleType::Bytes - } - RequestedModuleType::Other(text) => { - deno_resolver::loader::RequestedModuleType::Other(text) - } - } - } - let graph = self.graph_container.graph(); + let deno_resolver_requested_module_type = + as_deno_resolver_requested_module_type(requested_module_type); match self .shared .prepared_module_loader .load_prepared_module( &graph, specifier, - &as_deno_resolver_requested_module_type(requested_module_type), + &deno_resolver_requested_module_type, ) .await .map_err(LoadCodeSourceError::from)? { Some(module_or_asset) => match module_or_asset { - PreparedModuleOrAsset::Module(prepared_module) => { - Ok(ModuleCodeStringSource { - code: match prepared_module.source { - PreparedModuleSource::ArcStr(text) => { - ModuleSourceCode::String(text.into()) - } - PreparedModuleSource::ArcBytes(bytes) => { - ModuleSourceCode::Bytes(bytes.into()) - } - }, - found_url: prepared_module.specifier.clone(), - module_type: match requested_module_type { - RequestedModuleType::Json => ModuleType::Json, - RequestedModuleType::Text => ModuleType::Text, - RequestedModuleType::Bytes => ModuleType::Bytes, - RequestedModuleType::None | RequestedModuleType::Other(_) => { - module_type_from_media_type(prepared_module.media_type) - } - }, - }) + LoadedModuleOrAsset::Module(prepared_module) => { + Ok(self.loaded_module_to_module_code_string_source( + prepared_module, + requested_module_type, + )) } - PreparedModuleOrAsset::ExternalAsset { specifier } => { + LoadedModuleOrAsset::ExternalAsset { specifier } => { self.load_asset( specifier, /* do not use dynamic import permissions because this was statically analyzable */ CheckSpecifierKind::Static, @@ -812,13 +777,22 @@ impl } else { Cow::Borrowed(specifier) }; + if self.shared.in_npm_pkg_checker.in_npm_package(&specifier) { - return self + let loaded_module = self .shared .npm_module_loader - .load(&specifier, maybe_referrer) + .load( + &specifier, + maybe_referrer, + &deno_resolver_requested_module_type, + ) .await - .map_err(LoadCodeSourceError::from); + .map_err(LoadCodeSourceError::from)?; + return Ok(self.loaded_module_to_module_code_string_source( + loaded_module, + requested_module_type, + )); } match requested_module_type { @@ -839,6 +813,21 @@ impl } } + fn loaded_module_to_module_code_string_source( + &self, + loaded_module: LoadedModule, + requested_module_type: &RequestedModuleType, + ) -> ModuleCodeStringSource { + ModuleCodeStringSource { + code: loaded_module_source_to_module_source_code(loaded_module.source), + found_url: loaded_module.specifier.clone(), + module_type: module_type_from_media_and_requested_type( + loaded_module.media_type, + requested_module_type, + ), + } + } + async fn load_asset( &self, specifier: &ModuleSpecifier, diff --git a/cli/rt/node.rs b/cli/rt/node.rs index 774828b3cb..74a1343305 100644 --- a/cli/rt/node.rs +++ b/cli/rt/node.rs @@ -5,9 +5,9 @@ use std::sync::Arc; use deno_core::url::Url; use deno_error::JsErrorBox; -use deno_lib::loader::NpmModuleLoader; use deno_lib::standalone::binary::CjsExportAnalysisEntry; use deno_media_type::MediaType; +use deno_resolver::loader::NpmModuleLoader; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_resolver::npm::NpmReqResolver; use deno_runtime::deno_fs::FileSystem; diff --git a/cli/rt/run.rs b/cli/rt/run.rs index c581292f79..f24be16ea5 100644 --- a/cli/rt/run.rs +++ b/cli/rt/run.rs @@ -27,7 +27,9 @@ use deno_lib::args::CaData; use deno_lib::args::RootCertStoreLoadError; use deno_lib::args::get_root_cert_store; use deno_lib::args::npm_pkg_req_ref_to_binary_command; -use deno_lib::loader::NpmModuleLoader; +use deno_lib::loader::as_deno_resolver_requested_module_type; +use deno_lib::loader::loaded_module_source_to_module_source_code; +use deno_lib::loader::module_type_from_media_and_requested_type; use deno_lib::npm::NpmRegistryReadPermissionChecker; use deno_lib::npm::NpmRegistryReadPermissionCheckerMode; use deno_lib::npm::create_npm_process_state_provider; @@ -48,6 +50,7 @@ use deno_package_json::PackageJsonDepValue; use deno_resolver::DenoResolveErrorKind; use deno_resolver::cjs::CjsTracker; use deno_resolver::cjs::IsCjsResolutionMode; +use deno_resolver::loader::NpmModuleLoader; use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_resolver::npm::CreateInNpmPkgCheckerOptions; use deno_resolver::npm::DenoInNpmPackageChecker; @@ -404,18 +407,31 @@ impl ModuleLoader for EmbeddedModuleLoader { async move { let code_source = shared .npm_module_loader - .load(&original_specifier, maybe_referrer.as_ref()) + .load( + &original_specifier, + maybe_referrer.as_ref(), + &as_deno_resolver_requested_module_type(&requested_module_type), + ) .await .map_err(JsErrorBox::from_err)?; - let code_cache_entry = shared.get_code_cache( - &code_source.found_url, - code_source.code.as_bytes(), - ); + let code_cache_entry = match requested_module_type { + RequestedModuleType::None => shared.get_code_cache( + code_source.specifier, + code_source.source.as_bytes(), + ), + RequestedModuleType::Other(_) + | RequestedModuleType::Json + | RequestedModuleType::Text + | RequestedModuleType::Bytes => None, + }; Ok(deno_core::ModuleSource::new_with_redirect( - code_source.module_type, - code_source.code, + module_type_from_media_and_requested_type( + code_source.media_type, + &requested_module_type, + ), + loaded_module_source_to_module_source_code(code_source.source), &original_specifier, - &code_source.found_url, + code_source.specifier, code_cache_entry, )) } diff --git a/libs/package_json/lib.rs b/libs/package_json/lib.rs index 6805115967..3a6f78cc68 100644 --- a/libs/package_json/lib.rs +++ b/libs/package_json/lib.rs @@ -200,7 +200,7 @@ pub enum PackageJsonLoadError { source: serde_json::Error, }, #[error( - "\"exports\" cannot contains some keys starting with '.' and some not.\nThe exports object must either be an object of package subpath keys\nor an object of main entry condition name keys only." + "\"exports\" cannot contain some keys starting with '.' and some not.\nThe exports object must either be an object of package subpath keys\nor an object of main entry condition name keys only." )] #[class(type)] InvalidExports, diff --git a/libs/resolver/factory.rs b/libs/resolver/factory.rs index 97416875c1..c2b8acb3b7 100644 --- a/libs/resolver/factory.rs +++ b/libs/resolver/factory.rs @@ -61,6 +61,8 @@ use crate::deno_json::CompilerOptionsResolver; use crate::deno_json::CompilerOptionsResolverRc; use crate::import_map::WorkspaceExternalImportMapLoader; use crate::import_map::WorkspaceExternalImportMapLoaderRc; +use crate::loader::DenoNpmModuleLoaderRc; +use crate::loader::NpmModuleLoader; use crate::lockfile::LockfileLock; use crate::lockfile::LockfileLockRc; use crate::npm::ByonmNpmResolverCreateOptions; @@ -224,6 +226,7 @@ pub trait WorkspaceFactorySys: + crate::npm::NpmResolverSys + deno_cache_dir::GlobalHttpCacheSys + deno_cache_dir::LocalHttpCacheSys + + crate::loader::NpmModuleLoaderSys { } @@ -693,6 +696,7 @@ pub struct ResolverFactory { found_package_json_dep_flag: crate::graph::FoundPackageJsonDepFlagRc, in_npm_package_checker: Deferred, node_code_translator: Deferred>, + npm_module_loader: Deferred>, node_resolver: Deferred< NodeResolverRc< DenoInNpmPackageChecker, @@ -746,6 +750,7 @@ impl ResolverFactory { in_npm_package_checker: Default::default(), node_code_translator: Default::default(), node_resolver: Default::default(), + npm_module_loader: Default::default(), npm_req_resolver: Default::default(), npm_resolution: Default::default(), npm_resolver: Default::default(), @@ -949,6 +954,18 @@ impl ResolverFactory { }) } + pub fn npm_module_loader( + &self, + ) -> Result<&DenoNpmModuleLoaderRc, anyhow::Error> { + self.npm_module_loader.get_or_try_init(|| { + Ok(new_rc(NpmModuleLoader::new( + self.cjs_tracker()?.clone(), + self.node_code_translator()?.clone(), + self.workspace_factory.sys.clone(), + ))) + }) + } + pub fn npm_resolution(&self) -> &NpmResolutionCellRc { &self.npm_resolution } diff --git a/libs/resolver/loader/mod.rs b/libs/resolver/loader/mod.rs index 05e7ebffdb..a9815a02b2 100644 --- a/libs/resolver/loader/mod.rs +++ b/libs/resolver/loader/mod.rs @@ -1,5 +1,18 @@ // Copyright 2018-2025 the Deno authors. MIT license. +mod npm; + +#[cfg(all(feature = "graph", feature = "deno_ast"))] +mod prepared; + +use std::borrow::Cow; + +use deno_media_type::MediaType; +pub use npm::*; +#[cfg(all(feature = "graph", feature = "deno_ast"))] +pub use prepared::*; +use url::Url; + pub enum RequestedModuleType<'a> { None, Json, @@ -8,8 +21,31 @@ pub enum RequestedModuleType<'a> { Other(&'a str), } -#[cfg(all(feature = "graph", feature = "deno_ast"))] -mod prepared; +#[allow(clippy::disallowed_types)] +type ArcStr = std::sync::Arc; +#[allow(clippy::disallowed_types)] +type ArcBytes = std::sync::Arc<[u8]>; -#[cfg(all(feature = "graph", feature = "deno_ast"))] -pub use prepared::*; +pub struct LoadedModule<'a> { + pub specifier: &'a Url, + pub media_type: MediaType, + pub source: LoadedModuleSource, +} + +pub enum LoadedModuleSource { + ArcStr(ArcStr), + ArcBytes(ArcBytes), + String(Cow<'static, str>), + Bytes(Cow<'static, [u8]>), +} + +impl LoadedModuleSource { + pub fn as_bytes(&self) -> &[u8] { + match self { + LoadedModuleSource::ArcStr(text) => text.as_bytes(), + LoadedModuleSource::ArcBytes(bytes) => bytes, + LoadedModuleSource::String(text) => text.as_bytes(), + LoadedModuleSource::Bytes(bytes) => bytes, + } + } +} diff --git a/libs/resolver/loader/npm.rs b/libs/resolver/loader/npm.rs new file mode 100644 index 0000000000..e698945e63 --- /dev/null +++ b/libs/resolver/loader/npm.rs @@ -0,0 +1,251 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +use std::borrow::Cow; +use std::path::PathBuf; + +use deno_media_type::MediaType; +use node_resolver::InNpmPackageChecker; +use node_resolver::IsBuiltInNodeModuleChecker; +use node_resolver::NpmPackageFolderResolver; +use node_resolver::analyze::CjsCodeAnalyzer; +use node_resolver::analyze::NodeCodeTranslatorRc; +use node_resolver::analyze::NodeCodeTranslatorSys; +use thiserror::Error; +use url::Url; + +use super::LoadedModule; +use super::LoadedModuleSource; +use super::RequestedModuleType; +use crate::cjs::CjsTrackerRc; + +#[derive(Debug, Error, deno_error::JsError)] +#[class(type)] +#[error("[{}]: Stripping types is currently unsupported for files under node_modules, for \"{}\"", self.code(), specifier)] +pub struct StrippingTypesNodeModulesError { + pub specifier: Url, +} + +impl StrippingTypesNodeModulesError { + pub fn code(&self) -> &'static str { + "ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING" + } +} + +#[derive(Debug, Error, deno_error::JsError)] +pub enum NpmModuleLoadError { + #[class(inherit)] + #[error(transparent)] + UrlToFilePath(#[from] deno_path_util::UrlToFilePathError), + #[class(inherit)] + #[error(transparent)] + StrippingTypesNodeModules(#[from] StrippingTypesNodeModulesError), + #[class(inherit)] + #[error(transparent)] + ClosestPkgJson(#[from] node_resolver::errors::ClosestPkgJsonError), + #[class(inherit)] + #[error(transparent)] + TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError), + #[class(inherit)] + #[error("Unable to load {}{}", file_path.display(), maybe_referrer.as_ref().map(|r| format!(" imported from {}", r)).unwrap_or_default())] + UnableToLoad { + file_path: PathBuf, + maybe_referrer: Option, + #[source] + #[inherit] + source: std::io::Error, + }, + #[class(inherit)] + #[error( + "{}", + format_dir_import_message(file_path, maybe_referrer, suggestion) + )] + DirImport { + file_path: PathBuf, + maybe_referrer: Option, + suggestion: Option<&'static str>, + #[source] + #[inherit] + source: std::io::Error, + }, +} + +fn format_dir_import_message( + file_path: &std::path::Path, + maybe_referrer: &Option, + suggestion: &Option<&'static str>, +) -> String { + // directory imports are not allowed when importing from an + // ES module, so provide the user with a helpful error message + let dir_path = file_path; + let mut msg = "Directory import ".to_string(); + msg.push_str(&dir_path.to_string_lossy()); + if let Some(referrer) = maybe_referrer { + msg.push_str(" is not supported resolving import from "); + msg.push_str(referrer.as_str()); + if let Some(entrypoint_name) = suggestion { + msg.push_str("\nDid you mean to import "); + msg.push_str(entrypoint_name); + msg.push_str(" within the directory?"); + } + } + msg +} + +#[sys_traits::auto_impl] +pub trait NpmModuleLoaderSys: NodeCodeTranslatorSys {} + +#[allow(clippy::disallowed_types)] +pub type DenoNpmModuleLoaderRc = + crate::sync::MaybeArc>; + +pub type DenoNpmModuleLoader = NpmModuleLoader< + crate::cjs::analyzer::DenoCjsCodeAnalyzer, + crate::npm::DenoInNpmPackageChecker, + node_resolver::DenoIsBuiltInNodeModuleChecker, + crate::npm::NpmResolver, + TSys, +>; + +#[derive(Clone)] +pub struct NpmModuleLoader< + TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TSys: NpmModuleLoaderSys, +> { + cjs_tracker: CjsTrackerRc, + sys: TSys, + node_code_translator: NodeCodeTranslatorRc< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, +} + +impl< + TCjsCodeAnalyzer: CjsCodeAnalyzer, + TInNpmPackageChecker: InNpmPackageChecker, + TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver: NpmPackageFolderResolver, + TSys: NpmModuleLoaderSys, +> + NpmModuleLoader< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + > +{ + pub fn new( + cjs_tracker: CjsTrackerRc, + node_code_translator: NodeCodeTranslatorRc< + TCjsCodeAnalyzer, + TInNpmPackageChecker, + TIsBuiltInNodeModuleChecker, + TNpmPackageFolderResolver, + TSys, + >, + sys: TSys, + ) -> Self { + Self { + cjs_tracker, + node_code_translator, + sys, + } + } + + pub async fn load<'a>( + &self, + specifier: &'a Url, + maybe_referrer: Option<&Url>, + requested_module_type: &RequestedModuleType<'_>, + ) -> Result, NpmModuleLoadError> { + let file_path = deno_path_util::url_to_file_path(specifier)?; + let code = self.sys.fs_read(&file_path).map_err(|source| { + if self.sys.fs_is_dir_no_err(&file_path) { + let suggestion = ["index.mjs", "index.js", "index.cjs"] + .into_iter() + .find(|e| self.sys.fs_is_file_no_err(file_path.join(e))); + NpmModuleLoadError::DirImport { + file_path, + maybe_referrer: maybe_referrer.cloned(), + suggestion, + source, + } + } else { + NpmModuleLoadError::UnableToLoad { + file_path, + maybe_referrer: maybe_referrer.cloned(), + source, + } + } + })?; + + let media_type = MediaType::from_specifier(specifier); + match requested_module_type { + RequestedModuleType::Text | RequestedModuleType::Bytes => { + Ok(LoadedModule { + specifier, + media_type, + source: LoadedModuleSource::Bytes(code), + }) + } + RequestedModuleType::None + | RequestedModuleType::Json + | RequestedModuleType::Other(_) => { + if media_type.is_emittable() { + return Err(NpmModuleLoadError::StrippingTypesNodeModules( + StrippingTypesNodeModulesError { + specifier: specifier.clone(), + }, + )); + } + + let source = if self.cjs_tracker.is_maybe_cjs(specifier, media_type)? { + // translate cjs to esm if it's cjs and inject node globals + let code = from_utf8_lossy_cow(code); + LoadedModuleSource::String( + self + .node_code_translator + .translate_cjs_to_esm(specifier, Some(code)) + .await? + .into_owned() + .into(), + ) + } else { + // esm and json code is untouched + LoadedModuleSource::Bytes(code) + }; + + Ok(LoadedModule { + source, + specifier, + media_type, + }) + } + } + } +} + +#[inline(always)] +fn from_utf8_lossy_cow(bytes: Cow<[u8]>) -> Cow { + match bytes { + Cow::Borrowed(bytes) => String::from_utf8_lossy(bytes), + Cow::Owned(bytes) => Cow::Owned(from_utf8_lossy_owned(bytes)), + } +} + +// todo(https://github.com/rust-lang/rust/issues/129436): remove once stabilized +#[inline(always)] +fn from_utf8_lossy_owned(bytes: Vec) -> String { + match String::from_utf8_lossy(&bytes) { + Cow::Owned(code) => code, + // SAFETY: `String::from_utf8_lossy` guarantees that the result is valid + // UTF-8 if `Cow::Borrowed` is returned. + Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(bytes) }, + } +} diff --git a/libs/resolver/loader/prepared.rs b/libs/resolver/loader/prepared.rs index e9bd68ff03..11bfa0fc46 100644 --- a/libs/resolver/loader/prepared.rs +++ b/libs/resolver/loader/prepared.rs @@ -15,6 +15,8 @@ use node_resolver::errors::ClosestPkgJsonError; use thiserror::Error; use url::Url; +use super::LoadedModule; +use super::LoadedModuleSource; use super::RequestedModuleType; use crate::cache::ParsedSourceCacheRc; use crate::cjs::CjsTrackerRc; @@ -29,22 +31,6 @@ use crate::npm::NpmResolverSys; #[allow(clippy::disallowed_types)] type ArcStr = std::sync::Arc; -#[allow(clippy::disallowed_types)] -type ArcBytes = std::sync::Arc<[u8]>; - -pub enum PreparedModuleSource { - ArcStr(ArcStr), - ArcBytes(ArcBytes), -} - -impl PreparedModuleSource { - pub fn as_bytes(&self) -> &[u8] { - match self { - PreparedModuleSource::ArcStr(text) => text.as_bytes(), - PreparedModuleSource::ArcBytes(bytes) => bytes, - } - } -} #[derive(Debug, thiserror::Error, deno_error::JsError)] #[error("{message}")] @@ -91,14 +77,8 @@ pub trait PreparedModuleLoaderSys: { } -pub struct PreparedModule<'graph> { - pub specifier: &'graph Url, - pub media_type: MediaType, - pub source: PreparedModuleSource, -} - -pub enum PreparedModuleOrAsset<'graph> { - Module(PreparedModule<'graph>), +pub enum LoadedModuleOrAsset<'graph> { + Module(LoadedModule<'graph>), /// A module that the graph knows about, but the data /// is not stored in the graph itself. It's up to the caller /// to fetch this data. @@ -108,7 +88,7 @@ pub enum PreparedModuleOrAsset<'graph> { } enum CodeOrDeferredEmit<'a> { - Source(PreparedModule<'a>), + Source(LoadedModule<'a>), DeferredEmit { specifier: &'a Url, media_type: MediaType, @@ -159,8 +139,7 @@ impl graph: &'graph ModuleGraph, specifier: &Url, requested_module_type: &RequestedModuleType<'_>, - ) -> Result>, LoadPreparedModuleError> - { + ) -> Result>, LoadPreparedModuleError> { // Note: keep this in sync with the sync version below match self.load_prepared_module_or_defer_emit( graph, @@ -168,7 +147,7 @@ impl requested_module_type, )? { Some(CodeOrDeferredEmit::Source(source)) => { - Ok(Some(PreparedModuleOrAsset::Module(source))) + Ok(Some(LoadedModuleOrAsset::Module(source))) } Some(CodeOrDeferredEmit::DeferredEmit { specifier, @@ -183,9 +162,9 @@ impl // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(Some(PreparedModuleOrAsset::Module(PreparedModule { + Ok(Some(LoadedModuleOrAsset::Module(LoadedModule { // note: it's faster to provide a string to v8 if we know it's a string - source: PreparedModuleSource::ArcStr(transpile_result), + source: LoadedModuleSource::ArcStr(transpile_result), specifier, media_type, }))) @@ -198,15 +177,15 @@ impl .load_maybe_cjs(specifier, media_type, source) .await .map(|text| { - Some(PreparedModuleOrAsset::Module(PreparedModule { + Some(LoadedModuleOrAsset::Module(LoadedModule { specifier, media_type, - source: PreparedModuleSource::ArcStr(text), + source: LoadedModuleSource::ArcStr(text), })) }) .map_err(LoadPreparedModuleError::LoadMaybeCjs), Some(CodeOrDeferredEmit::ExternalAsset { specifier }) => { - Ok(Some(PreparedModuleOrAsset::ExternalAsset { specifier })) + Ok(Some(LoadedModuleOrAsset::ExternalAsset { specifier })) } None => Ok(None), } @@ -216,7 +195,7 @@ impl &self, graph: &'graph ModuleGraph, specifier: &Url, - ) -> Result>, anyhow::Error> { + ) -> Result>, anyhow::Error> { // Note: keep this in sync with the async version above match self.load_prepared_module_or_defer_emit( graph, @@ -239,9 +218,9 @@ impl // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(Some(PreparedModule { + Ok(Some(LoadedModule { // note: it's faster to provide a string if we know it's a string - source: PreparedModuleSource::ArcStr(transpile_result), + source: LoadedModuleSource::ArcStr(transpile_result), specifier, media_type, })) @@ -282,22 +261,22 @@ impl .. })) => match requested_module_type { RequestedModuleType::Bytes => match source.try_get_original_bytes() { - Some(bytes) => Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcBytes(bytes), + Some(bytes) => Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcBytes(bytes), specifier, media_type: *media_type, }))), None => Ok(Some(CodeOrDeferredEmit::ExternalAsset { specifier })), }, RequestedModuleType::Text => { - Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcStr(source.text.clone()), + Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcStr(source.text.clone()), specifier, media_type: *media_type, }))) } - _ => Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcStr(source.text.clone()), + _ => Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcStr(source.text.clone()), specifier, media_type: *media_type, }))), @@ -310,16 +289,16 @@ impl .. })) => match requested_module_type { RequestedModuleType::Bytes => match source.try_get_original_bytes() { - Some(bytes) => Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcBytes(bytes), + Some(bytes) => Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcBytes(bytes), specifier, media_type: *media_type, }))), None => Ok(Some(CodeOrDeferredEmit::ExternalAsset { specifier })), }, RequestedModuleType::Text => { - Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcStr(source.text.clone()), + Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcStr(source.text.clone()), specifier, media_type: *media_type, }))) @@ -373,8 +352,8 @@ impl // at this point, we no longer need the parsed source in memory, so free it self.parsed_source_cache.free(specifier); - Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcStr(code), + Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcStr(code), specifier, media_type: *media_type, }))) @@ -382,8 +361,8 @@ impl }, Some(deno_graph::Module::Wasm(WasmModule { source, specifier, .. - })) => Ok(Some(CodeOrDeferredEmit::Source(PreparedModule { - source: PreparedModuleSource::ArcBytes(source.clone()), + })) => Ok(Some(CodeOrDeferredEmit::Source(LoadedModule { + source: LoadedModuleSource::ArcBytes(source.clone()), specifier, media_type: MediaType::Wasm, }))), diff --git a/tests/specs/npm/bytes_and_text_imports/__test__.jsonc b/tests/specs/npm/bytes_and_text_imports/__test__.jsonc new file mode 100644 index 0000000000..fa0e32b4fb --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/__test__.jsonc @@ -0,0 +1,19 @@ +{ + "tempDir": true, + "tests": { + "run": { + "args": "run main.ts", + "output": "run.out" + }, + "compile": { + "steps": [{ + "args": "compile --unstable-raw-imports --output bin main.ts", + "output": "[WILDCARD]" + }, { + "commandName": "./bin", + "args": [], + "output": "run.out" + }] + } + } +} diff --git a/tests/specs/npm/bytes_and_text_imports/deno.json b/tests/specs/npm/bytes_and_text_imports/deno.json new file mode 100644 index 0000000000..8af304e245 --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/deno.json @@ -0,0 +1,5 @@ +{ + "unstable": [ + "raw-imports" + ] +} diff --git a/tests/specs/npm/bytes_and_text_imports/main.ts b/tests/specs/npm/bytes_and_text_imports/main.ts new file mode 100644 index 0000000000..7a61441cf2 --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/main.ts @@ -0,0 +1,9 @@ +import bytes from "package/style.css" with { type: "bytes" }; +import text from "package/style.css" with { type: "text" }; +console.log(bytes); +console.log(text); + +import bytesUtf8Bom from "package/style_utf8_bom.css" with { type: "bytes" }; +import textUtf8Bom from "package/style_utf8_bom.css" with { type: "text" }; +console.log(bytesUtf8Bom); +console.log(textUtf8Bom); diff --git a/tests/specs/npm/bytes_and_text_imports/node_modules/package/package.json b/tests/specs/npm/bytes_and_text_imports/node_modules/package/package.json new file mode 100644 index 0000000000..920dda0800 --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/node_modules/package/package.json @@ -0,0 +1,3 @@ +{ + "name": "package" +} diff --git a/tests/specs/npm/bytes_and_text_imports/node_modules/package/style.css b/tests/specs/npm/bytes_and_text_imports/node_modules/package/style.css new file mode 100644 index 0000000000..a9f2a4ff6a --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/node_modules/package/style.css @@ -0,0 +1,3 @@ +div { + border-color: green; +} diff --git a/tests/specs/npm/bytes_and_text_imports/node_modules/package/style_utf8_bom.css b/tests/specs/npm/bytes_and_text_imports/node_modules/package/style_utf8_bom.css new file mode 100644 index 0000000000..1f3f9147fb --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/node_modules/package/style_utf8_bom.css @@ -0,0 +1,3 @@ +div { + border-color: green; +} diff --git a/tests/specs/npm/bytes_and_text_imports/package.json b/tests/specs/npm/bytes_and_text_imports/package.json new file mode 100644 index 0000000000..2c63c08510 --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/package.json @@ -0,0 +1,2 @@ +{ +} diff --git a/tests/specs/npm/bytes_and_text_imports/run.out b/tests/specs/npm/bytes_and_text_imports/run.out new file mode 100644 index 0000000000..781dfa0a99 --- /dev/null +++ b/tests/specs/npm/bytes_and_text_imports/run.out @@ -0,0 +1,22 @@ +Uint8Array(31) [ + 100, 105, 118, 32, 123, 10, 32, + 32, 98, 111, 114, 100, 101, 114, + 45, 99, 111, 108, 111, 114, 58, + 32, 103, 114, 101, 101, 110, 59, + 10, 125, 10 +] +div { + border-color: green; +} + +Uint8Array(34) [ + 239, 187, 191, 100, 105, 118, 32, 123, + 10, 32, 32, 98, 111, 114, 100, 101, + 114, 45, 99, 111, 108, 111, 114, 58, + 32, 103, 114, 101, 101, 110, 59, 10, + 125, 10 +] +div { + border-color: green; +} +