From 7a1e949c476a3b55d6fb8a1141f74bc96c9da8fb Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Jul 2025 08:56:02 -0400 Subject: [PATCH] refactor: extract `DenoDir`, `Emitter`, `EmitCache` and `ParsedSourceCache` from cli crate (#29961) --- cli/cache/mod.rs | 13 +- cli/cache/module_info.rs | 2 +- cli/factory.rs | 97 +++------ cli/graph_util.rs | 2 +- cli/lsp/config.rs | 6 +- cli/main.rs | 1 - cli/module_loader.rs | 39 ++-- cli/node.rs | 2 +- cli/standalone/binary.rs | 6 +- cli/tools/compile.rs | 2 +- cli/tools/doc.rs | 2 +- cli/tools/publish/graph.rs | 2 +- cli/tools/publish/mod.rs | 5 +- cli/tools/publish/module_content.rs | 4 +- cli/tools/run/hmr.rs | 12 +- libs/resolver/Cargo.toml | 4 +- {cli => libs/resolver}/cache/deno_dir.rs | 59 ++++-- {cli => libs/resolver}/cache/disk_cache.rs | 63 +++--- {cli => libs/resolver}/cache/emit.rs | 86 ++++---- libs/resolver/cache/mod.rs | 24 +++ {cli => libs/resolver}/cache/parsed_source.rs | 83 ++++---- {cli => libs/resolver}/emit.rs | 196 ++++++++++-------- libs/resolver/factory.rs | 144 +++++++------ libs/resolver/lib.rs | 5 + libs/resolver/rt.rs | 24 +++ libs/resolver/sync.rs | 13 ++ 26 files changed, 490 insertions(+), 406 deletions(-) rename {cli => libs/resolver}/cache/deno_dir.rs (75%) rename {cli => libs/resolver}/cache/disk_cache.rs (92%) rename {cli => libs/resolver}/cache/emit.rs (79%) create mode 100644 libs/resolver/cache/mod.rs rename {cli => libs/resolver}/cache/parsed_source.rs (65%) rename {cli => libs/resolver}/emit.rs (75%) create mode 100644 libs/resolver/rt.rs diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 84532e4ece..43247850bb 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -4,14 +4,13 @@ mod cache_db; mod caches; mod check; mod code_cache; -mod deno_dir; -mod disk_cache; -mod emit; mod fast_check; mod incremental; mod module_info; mod node; -mod parsed_source; + +pub type DenoDir = deno_resolver::cache::DenoDir; +pub type DenoDirProvider = deno_resolver::cache::DenoDirProvider; pub use cache_db::CacheDBHash; pub use caches::Caches; @@ -19,16 +18,10 @@ pub use check::TypeCheckCache; pub use code_cache::CodeCache; /// Permissions used to save a file in the disk caches. pub use deno_cache_dir::CACHE_PERM; -pub use deno_dir::DenoDir; -pub use deno_dir::DenoDirProvider; -pub use disk_cache::DiskCache; -pub use emit::EmitCache; pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; pub use module_info::ModuleInfoCache; pub use node::NodeAnalysisCache; -pub use parsed_source::LazyGraphSourceParser; -pub use parsed_source::ParsedSourceCache; use crate::sys::CliSys; diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs index 25c0a4b4d0..4a57fd956d 100644 --- a/cli/cache/module_info.rs +++ b/cli/cache/module_info.rs @@ -9,13 +9,13 @@ use deno_core::serde_json; use deno_error::JsErrorBox; use deno_graph::analysis::ModuleInfo; use deno_graph::ast::ParserModuleAnalyzer; +use deno_resolver::cache::ParsedSourceCache; use deno_runtime::deno_webstorage::rusqlite::params; use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::cache_db::CacheDBHash; use super::cache_db::CacheFailure; -use super::ParsedSourceCache; const SELECT_MODULE_INFO: &str = " SELECT diff --git a/cli/factory.rs b/cli/factory.rs index edc2ab2909..87aa53a8d5 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -31,10 +31,10 @@ use deno_npm_installer::lifecycle_scripts::LifecycleScriptsExecutor; use deno_npm_installer::lifecycle_scripts::NullLifecycleScriptsExecutor; use deno_npm_installer::process_state::NpmProcessStateKind; use deno_npm_installer::NpmInstallerFactoryOptions; +use deno_resolver::cache::ParsedSourceCache; use deno_resolver::cjs::IsCjsResolutionMode; use deno_resolver::deno_json::CompilerOptionsResolver; use deno_resolver::factory::ConfigDiscoveryOption; -use deno_resolver::factory::DenoDirPathProviderOptions; use deno_resolver::factory::NpmProcessStateOptions; use deno_resolver::factory::ResolverFactoryOptions; use deno_resolver::factory::SpecifiedImportMapProvider; @@ -70,13 +70,9 @@ use crate::args::InstallFlags; use crate::cache::Caches; use crate::cache::CodeCache; use crate::cache::DenoDir; -use crate::cache::DenoDirProvider; -use crate::cache::EmitCache; use crate::cache::GlobalHttpCache; use crate::cache::ModuleInfoCache; use crate::cache::NodeAnalysisCache; -use crate::cache::ParsedSourceCache; -use crate::emit::Emitter; use crate::file_fetcher::create_cli_file_fetcher; use crate::file_fetcher::CliFileFetcher; use crate::file_fetcher::CreateCliFileFetcherOptions; @@ -86,6 +82,7 @@ use crate::graph_util::FileWatcherReporter; use crate::graph_util::ModuleGraphBuilder; use crate::graph_util::ModuleGraphCreator; use crate::http_util::HttpClientProvider; +use crate::module_loader::CliEmitter; use crate::module_loader::CliModuleLoaderFactory; use crate::module_loader::EszipModuleLoader; use crate::module_loader::ModuleLoadPreparer; @@ -249,9 +246,6 @@ impl SpecifiedImportMapProvider for CliSpecifiedImportMapProvider { } pub type CliWorkspaceFactory = deno_resolver::factory::WorkspaceFactory; -pub type CliDenoDirPathProvider = - deno_resolver::factory::DenoDirPathProvider; - pub type CliResolverFactory = deno_resolver::factory::ResolverFactory; pub struct Deferred(once_cell::unsync::OnceCell); @@ -307,10 +301,6 @@ struct CliFactoryServices { cjs_module_export_analyzer: Deferred>, cli_options: Deferred>, code_cache: Deferred>, - deno_dir_path_provider: Deferred>, - deno_dir_provider: Deferred>, - emit_cache: Deferred>, - emitter: Deferred>, eszip_module_loader_provider: Deferred>, feature_checker: Deferred>, file_fetcher: Deferred>, @@ -325,7 +315,6 @@ struct CliFactoryServices { module_load_preparer: Deferred>, node_code_translator: Deferred>, npm_installer_factory: Deferred, - parsed_source_cache: Deferred>, permission_desc_parser: Deferred>>, resolver_factory: Deferred>, @@ -398,34 +387,21 @@ impl CliFactory { }) } - pub fn deno_dir_path_provider(&self) -> &Arc { - self.services.deno_dir_path_provider.get_or_init(|| { - Arc::new(CliDenoDirPathProvider::new( - self.sys(), - DenoDirPathProviderOptions { - maybe_custom_root: self.flags.internal.cache_path.clone(), - }, - )) - }) - } - - pub fn deno_dir_provider(&self) -> &Arc { - self.services.deno_dir_provider.get_or_init(|| { - Arc::new(DenoDirProvider::new( - self.sys(), - self.deno_dir_path_provider().clone(), - )) - }) - } - pub fn deno_dir(&self) -> Result<&DenoDir, AnyError> { - Ok(self.deno_dir_provider().get_or_create()?) + Ok( + self + .workspace_factory()? + .deno_dir_provider() + .get_or_create()?, + ) } pub fn caches(&self) -> Result<&Arc, AnyError> { self.services.caches.get_or_try_init(|| { let cli_options = self.cli_options()?; - let caches = Arc::new(Caches::new(self.deno_dir_provider().clone())); + let caches = Arc::new(Caches::new( + self.workspace_factory()?.deno_dir_provider().clone(), + )); // Warm up the caches we know we'll likely need based on the CLI mode match cli_options.sub_command() { DenoSubcommand::Run(_) @@ -623,11 +599,7 @@ impl CliFactory { .env_current_dir() .with_context(|| "Failed getting cwd.")?, }; - let options = new_workspace_factory_options( - &initial_cwd, - &self.flags, - self.deno_dir_path_provider().clone(), - ); + let options = new_workspace_factory_options(&initial_cwd, &self.flags); let mut factory = CliWorkspaceFactory::new(self.sys(), initial_cwd, options); if let Some(workspace_dir) = &self.overrides.workspace_directory { @@ -660,17 +632,11 @@ impl CliFactory { .get_or_init(|| maybe_file_watcher_reporter) } - pub fn emit_cache(&self) -> Result<&Arc, AnyError> { - self.services.emit_cache.get_or_try_init(|| { - Ok(Arc::new(EmitCache::new(self.deno_dir()?.gen_cache.clone()))) - }) - } - pub fn module_info_cache(&self) -> Result<&Arc, AnyError> { self.services.module_info_cache.get_or_try_init(|| { Ok(Arc::new(ModuleInfoCache::new( self.caches()?.dep_analysis_db(), - self.parsed_source_cache().clone(), + self.resolver_factory()?.parsed_source_cache().clone(), ))) }) } @@ -681,22 +647,14 @@ impl CliFactory { }) } - pub fn parsed_source_cache(&self) -> &Arc { - self - .services - .parsed_source_cache - .get_or_init(Default::default) + pub fn parsed_source_cache( + &self, + ) -> Result<&Arc, AnyError> { + Ok(self.resolver_factory()?.parsed_source_cache()) } - pub fn emitter(&self) -> Result<&Arc, AnyError> { - self.services.emitter.get_or_try_init(|| { - Ok(Arc::new(Emitter::new( - self.cjs_tracker()?.clone(), - self.emit_cache()?.clone(), - self.parsed_source_cache().clone(), - self.compiler_options_resolver()?.clone(), - ))) - }) + pub fn emitter(&self) -> Result<&Arc, AnyError> { + self.resolver_factory()?.emitter() } pub async fn lint_rule_provider(&self) -> Result { @@ -771,7 +729,7 @@ impl CliFactory { node_analysis_cache, self.cjs_tracker()?.clone(), self.fs().clone(), - Some(self.parsed_source_cache().clone()), + Some(self.parsed_source_cache()?.clone()), )) } @@ -841,7 +799,7 @@ impl CliFactory { self.npm_graph_resolver().await?.clone(), self.npm_installer_if_managed().await?.cloned(), self.npm_resolver().await?.clone(), - self.parsed_source_cache().clone(), + self.resolver_factory()?.parsed_source_cache().clone(), self.resolver().await?.clone(), self.root_permissions_container()?.clone(), self.sys(), @@ -1021,8 +979,9 @@ impl CliFactory { let node_code_translator = self.node_code_translator().await?; let cjs_tracker = self.cjs_tracker()?.clone(); let workspace_factory = self.workspace_factory()?; + let resolver_factory = self.resolver_factory()?; let npm_registry_permission_checker = { - let mode = if self.resolver_factory()?.use_byonm()? { + let mode = if resolver_factory.use_byonm()? { NpmRegistryReadPermissionCheckerMode::Byonm } else if let Some(node_modules_dir) = workspace_factory.node_modules_dir_path()? @@ -1061,7 +1020,7 @@ impl CliFactory { ), npm_registry_permission_checker, cli_npm_resolver.clone(), - self.parsed_source_cache().clone(), + resolver_factory.parsed_source_cache().clone(), self.resolver().await?.clone(), self.sys(), maybe_eszip_loader, @@ -1292,8 +1251,7 @@ impl CliFactory { fn new_workspace_factory_options( initial_cwd: &Path, flags: &Flags, - deno_dir_path_provider: Arc, -) -> deno_resolver::factory::WorkspaceFactoryOptions { +) -> deno_resolver::factory::WorkspaceFactoryOptions { deno_resolver::factory::WorkspaceFactoryOptions { additional_config_file_names: if matches!( flags.subcommand, @@ -1316,7 +1274,10 @@ fn new_workspace_factory_options( } ConfigFlag::Disabled => ConfigDiscoveryOption::Disabled, }, - deno_dir_path_provider: Some(deno_dir_path_provider), + emit_cache_version: Cow::Borrowed( + deno_lib::version::DENO_VERSION_INFO.deno, + ), + maybe_custom_deno_dir_root: flags.internal.cache_path.clone(), // For `deno install/add/remove/init` we want to force the managed // resolver so it can set up the `node_modules/` directory. is_package_manager_subcommand: matches!( diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 12b2afc6bb..4c6fc8ab64 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -34,6 +34,7 @@ use deno_graph::WorkspaceFastCheckOption; use deno_npm_installer::graph::NpmCachingStrategy; use deno_npm_installer::PackageCaching; use deno_path_util::url_to_file_path; +use deno_resolver::cache::ParsedSourceCache; use deno_resolver::deno_json::CompilerOptionsResolver; use deno_resolver::deno_json::JsxImportSourceConfigResolver; use deno_resolver::npm::DenoInNpmPackageChecker; @@ -53,7 +54,6 @@ use crate::args::DenoSubcommand; use crate::cache; use crate::cache::GlobalHttpCache; use crate::cache::ModuleInfoCache; -use crate::cache::ParsedSourceCache; use crate::colors; use crate::file_fetcher::CliDenoGraphLoader; use crate::file_fetcher::CliFileFetcher; diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 2ad9484cf6..baea43b67b 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -1,5 +1,6 @@ // Copyright 2018-2025 the Deno authors. MIT license. +use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; @@ -1451,8 +1452,11 @@ impl ConfigData { WorkspaceFactoryOptions { additional_config_file_names: &[], config_discovery: ConfigDiscoveryOption::DiscoverCwd, - deno_dir_path_provider: None, + maybe_custom_deno_dir_root: None, is_package_manager_subcommand: false, + emit_cache_version: Cow::Borrowed( + deno_lib::version::DENO_VERSION_INFO.deno, + ), frozen_lockfile: None, lock_arg: None, lockfile_skip_write: false, diff --git a/cli/main.rs b/cli/main.rs index 602db02033..a00d2a0348 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -3,7 +3,6 @@ mod args; mod cache; mod cdp; -mod emit; mod factory; mod file_fetcher; mod graph_container; diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 0725645a4a..3f173532be 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -54,6 +54,7 @@ use deno_lib::npm::NpmRegistryReadPermissionChecker; use deno_lib::util::hash::FastInsecureHasher; use deno_lib::worker::CreateModuleLoaderResult; use deno_lib::worker::ModuleLoaderFactory; +use deno_resolver::cache::ParsedSourceCache; use deno_resolver::file_fetcher::FetchOptions; use deno_resolver::file_fetcher::FetchPermissionsOptionRef; use deno_resolver::graph::ResolveWithGraphErrorKind; @@ -84,8 +85,6 @@ use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::TsTypeLib; use crate::cache::CodeCache; -use crate::cache::ParsedSourceCache; -use crate::emit::Emitter; use crate::file_fetcher::CliFileFetcher; use crate::graph_container::MainModuleGraphContainer; use crate::graph_container::ModuleGraphContainer; @@ -115,6 +114,8 @@ pub type CliNpmModuleLoader = deno_lib::loader::NpmModuleLoader< CliNpmResolver, CliSys, >; +pub type CliEmitter = + deno_resolver::emit::Emitter; #[derive(Debug, thiserror::Error, deno_error::JsError)] pub enum PrepareModuleLoadError { @@ -331,7 +332,7 @@ struct SharedCliModuleLoaderState { is_repl: bool, cjs_tracker: Arc, code_cache: Option>, - emitter: Arc, + emitter: Arc, file_fetcher: Arc, in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, @@ -393,7 +394,7 @@ impl CliModuleLoaderFactory { options: &CliOptions, cjs_tracker: Arc, code_cache: Option>, - emitter: Arc, + emitter: Arc, file_fetcher: Arc, in_npm_pkg_checker: DenoInNpmPackageChecker, main_module_graph_container: Arc, @@ -459,9 +460,6 @@ impl CliModuleLoaderFactory { parent_permissions, permissions, graph_container: graph_container.clone(), - node_code_translator: self.shared.node_code_translator.clone(), - emitter: self.shared.emitter.clone(), - parsed_source_cache: self.shared.parsed_source_cache.clone(), shared: self.shared.clone(), loaded_files: Default::default(), }))); @@ -527,9 +525,6 @@ impl CliModuleLoaderFactory { parent_permissions: root_permissions.clone(), permissions: root_permissions, graph_container: (*self.shared.main_module_graph_container).clone(), - node_code_translator: self.shared.node_code_translator.clone(), - emitter: self.shared.emitter.clone(), - parsed_source_cache: self.shared.parsed_source_cache.clone(), shared: self.shared.clone(), loaded_files: Default::default(), })) @@ -557,7 +552,7 @@ pub struct EnhancedGraphError { pub enum LoadPreparedModuleError { #[class(inherit)] #[error(transparent)] - NpmModuleLoad(#[from] crate::emit::EmitParsedSourceHelperError), + NpmModuleLoad(#[from] deno_resolver::emit::EmitParsedSourceHelperError), #[class(inherit)] #[error(transparent)] LoadMaybeCjs(#[from] LoadMaybeCjsError), @@ -576,7 +571,7 @@ pub enum LoadPreparedModuleError { pub enum LoadMaybeCjsError { #[class(inherit)] #[error(transparent)] - NpmModuleLoad(#[from] crate::emit::EmitParsedSourceHelperError), + NpmModuleLoad(#[from] deno_resolver::emit::EmitParsedSourceHelperError), #[class(inherit)] #[error(transparent)] TranslateCjsToEsm(#[from] node_resolver::analyze::TranslateCjsToEsmError), @@ -591,9 +586,6 @@ struct CliModuleLoaderInner { parent_permissions: PermissionsContainer, permissions: PermissionsContainer, shared: Arc, - emitter: Arc, - node_code_translator: Arc, - parsed_source_cache: Arc, graph_container: TGraphContainer, loaded_files: RefCell>, } @@ -1109,12 +1101,13 @@ impl source, }) => { let transpile_result = self + .shared .emitter .emit_parsed_source(specifier, media_type, ModuleKind::Esm, source) .await?; // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); + self.shared.parsed_source_cache.free(specifier); Ok(Some(ModuleCodeStringSource { // note: it's faster to provide a string if we know it's a string @@ -1162,7 +1155,7 @@ impl media_type, source, }) => { - let transpile_result = self.emitter.emit_parsed_source_sync( + let transpile_result = self.shared.emitter.emit_parsed_source_sync( specifier, media_type, ModuleKind::Esm, @@ -1170,7 +1163,7 @@ impl )?; // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); + self.shared.parsed_source_cache.free(specifier); Ok(Some(ModuleCodeStringSource { // note: it's faster to provide a string if we know it's a string @@ -1180,7 +1173,7 @@ impl })) } Some(CodeOrDeferredEmit::Cjs { .. }) => { - self.parsed_source_cache.free(specifier); + self.shared.parsed_source_cache.free(specifier); // todo(dsherret): to make this work, we should probably just // rely on the CJS export cache. At the moment this is hard because @@ -1316,7 +1309,7 @@ impl }; // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); + self.shared.parsed_source_cache.free(specifier); Ok(Some(CodeOrDeferredEmit::Code(ModuleCodeStringSource { code: ModuleSourceCode::String(code), @@ -1365,6 +1358,7 @@ impl let js_source = if media_type.is_emittable() { Cow::Owned( self + .shared .emitter .emit_parsed_source( specifier, @@ -1378,11 +1372,12 @@ impl Cow::Borrowed(original_source.as_ref()) }; let text = self + .shared .node_code_translator .translate_cjs_to_esm(specifier, Some(js_source)) .await?; // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); + self.shared.parsed_source_cache.free(specifier); Ok(ModuleCodeStringSource { code: match text { // perf: if the text is borrowed, that means it didn't make any changes @@ -1731,7 +1726,7 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { #[derive(Debug)] struct CliNodeRequireLoader { cjs_tracker: Arc, - emitter: Arc, + emitter: Arc, npm_resolver: CliNpmResolver, sys: CliSys, graph_container: TGraphContainer, diff --git a/cli/node.rs b/cli/node.rs index 3b80b04263..d451bb03fe 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -8,6 +8,7 @@ use deno_ast::ModuleExportsAndReExports; use deno_ast::ModuleSpecifier; use deno_error::JsErrorBox; use deno_graph::ast::ParsedSourceStore; +use deno_resolver::cache::ParsedSourceCache; use deno_resolver::npm::DenoInNpmPackageChecker; use deno_runtime::deno_fs; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -22,7 +23,6 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; -use crate::cache::ParsedSourceCache; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; use crate::sys::CliSys; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 799f745d47..e749d70d5c 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -64,9 +64,9 @@ use crate::args::get_default_v8_flags; use crate::args::CliOptions; use crate::args::CompileFlags; use crate::cache::DenoDir; -use crate::emit::Emitter; use crate::file_fetcher::CliFileFetcher; use crate::http_util::HttpClientProvider; +use crate::module_loader::CliEmitter; use crate::node::CliCjsModuleExportAnalyzer; use crate::npm::CliNpmResolver; use crate::resolver::CliCjsTracker; @@ -201,7 +201,7 @@ pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, deno_dir: &'a DenoDir, - emitter: &'a Emitter, + emitter: &'a CliEmitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a CliNpmResolver, @@ -216,7 +216,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CliCjsTracker, cli_options: &'a CliOptions, deno_dir: &'a DenoDir, - emitter: &'a Emitter, + emitter: &'a CliEmitter, file_fetcher: &'a CliFileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a CliNpmResolver, diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs index 1bf28aea2f..a1e815a010 100644 --- a/cli/tools/compile.rs +++ b/cli/tools/compile.rs @@ -162,7 +162,7 @@ pub async fn compile_eszip( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let module_graph_creator = factory.module_graph_creator().await?; - let parsed_source_cache = factory.parsed_source_cache(); + let parsed_source_cache = factory.parsed_source_cache()?; let compiler_options_resolver = factory.compiler_options_resolver()?; let bin_name_resolver = factory.bin_name_resolver()?; let entrypoint = cli_options.resolve_main_module()?; diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index d2e4533549..a233574f18 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -109,7 +109,7 @@ pub async fn doc( let factory = CliFactory::from_flags(flags); let cli_options = factory.cli_options()?; let module_info_cache = factory.module_info_cache()?; - let parsed_source_cache = factory.parsed_source_cache(); + let parsed_source_cache = factory.parsed_source_cache()?; let capturing_parser = parsed_source_cache.as_capturing_parser(); let analyzer = module_info_cache.as_module_analyzer(); diff --git a/cli/tools/publish/graph.rs b/cli/tools/publish/graph.rs index caa949b675..c23ccad023 100644 --- a/cli/tools/publish/graph.rs +++ b/cli/tools/publish/graph.rs @@ -13,12 +13,12 @@ use deno_graph::ModuleEntryRef; use deno_graph::ModuleGraph; use deno_graph::ResolutionResolved; use deno_graph::WalkOptions; +use deno_resolver::cache::ParsedSourceCache; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; use super::diagnostics::PublishDiagnostic; use super::diagnostics::PublishDiagnosticsCollector; -use crate::cache::ParsedSourceCache; pub struct GraphDiagnosticsCollector { parsed_source_cache: Arc, diff --git a/cli/tools/publish/mod.rs b/cli/tools/publish/mod.rs index c6a14b7d30..a38ffb7a4e 100644 --- a/cli/tools/publish/mod.rs +++ b/cli/tools/publish/mod.rs @@ -117,14 +117,15 @@ pub async fn publish( ); let diagnostics_collector = PublishDiagnosticsCollector::default(); + let parsed_source_cache = cli_factory.parsed_source_cache()?; let module_content_provider = Arc::new(ModuleContentProvider::new( - cli_factory.parsed_source_cache().clone(), + parsed_source_cache.clone(), specifier_unfurler, cli_factory.sys(), cli_factory.compiler_options_resolver()?.clone(), )); let publish_preparer = PublishPreparer::new( - GraphDiagnosticsCollector::new(cli_factory.parsed_source_cache().clone()), + GraphDiagnosticsCollector::new(parsed_source_cache.clone()), cli_factory.module_graph_creator().await?.clone(), cli_factory.type_checker().await?.clone(), cli_options.clone(), diff --git a/cli/tools/publish/module_content.rs b/cli/tools/publish/module_content.rs index 0ecfc8982d..ae800bb74b 100644 --- a/cli/tools/publish/module_content.rs +++ b/cli/tools/publish/module_content.rs @@ -11,6 +11,8 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::url::Url; use deno_graph::ModuleGraph; +use deno_resolver::cache::LazyGraphSourceParser; +use deno_resolver::cache::ParsedSourceCache; use deno_resolver::deno_json::CompilerOptionsResolver; use deno_resolver::workspace::ResolutionKind; use lazy_regex::Lazy; @@ -21,8 +23,6 @@ use super::diagnostics::PublishDiagnostic; use super::diagnostics::PublishDiagnosticsCollector; use super::unfurl::SpecifierUnfurler; use super::unfurl::SpecifierUnfurlerDiagnostic; -use crate::cache::LazyGraphSourceParser; -use crate::cache::ParsedSourceCache; use crate::sys::CliSys; struct JsxFolderOptions<'a> { diff --git a/cli/tools/run/hmr.rs b/cli/tools/run/hmr.rs index 0b4491e4fa..f0d3ae225f 100644 --- a/cli/tools/run/hmr.rs +++ b/cli/tools/run/hmr.rs @@ -15,7 +15,7 @@ use deno_terminal::colors; use tokio::select; use crate::cdp; -use crate::emit::Emitter; +use crate::module_loader::CliEmitter; use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherRestartMode; @@ -79,7 +79,7 @@ pub struct HmrRunner { session: LocalInspectorSession, watcher_communicator: Arc, script_ids: HashMap, - emitter: Arc, + emitter: Arc, } #[async_trait::async_trait(?Send)] @@ -159,9 +159,11 @@ impl crate::worker::HmrRunner for HmrRunner { continue; }; - let source_code = self.emitter.load_and_emit_for_hmr( + let source_code = tokio::fs::read_to_string(deno_path_util::url_to_file_path(&module_url).unwrap()).await?; + let source_code = self.emitter.emit_for_hmr( &module_url, - ).await?; + source_code, + )?; let mut tries = 1; loop { @@ -193,7 +195,7 @@ impl crate::worker::HmrRunner for HmrRunner { impl HmrRunner { pub fn new( - emitter: Arc, + emitter: Arc, session: LocalInspectorSession, watcher_communicator: Arc, ) -> Self { diff --git a/libs/resolver/Cargo.toml b/libs/resolver/Cargo.toml index 1d237c3626..6214165604 100644 --- a/libs/resolver/Cargo.toml +++ b/libs/resolver/Cargo.toml @@ -14,7 +14,7 @@ description = "Deno resolution algorithm" path = "lib.rs" [features] -deno_ast = ["dep:deno_ast", "twox-hash"] +deno_ast = ["dep:deno_ast", "deno_graph/swc"] graph = ["deno_graph", "node_resolver/graph", "http", "deno_permissions"] sync = ["dashmap", "deno_package_json/sync", "node_resolver/sync", "deno_config/sync", "deno_cache_dir/sync"] @@ -53,7 +53,7 @@ serde.workspace = true serde_json.workspace = true sys_traits.workspace = true thiserror.workspace = true -twox-hash = { workspace = true, optional = true } +twox-hash.workspace = true url.workspace = true [dev-dependencies] diff --git a/cli/cache/deno_dir.rs b/libs/resolver/cache/deno_dir.rs similarity index 75% rename from cli/cache/deno_dir.rs rename to libs/resolver/cache/deno_dir.rs index a5dd96ea7d..79a0b4a5c8 100644 --- a/cli/cache/deno_dir.rs +++ b/libs/resolver/cache/deno_dir.rs @@ -2,38 +2,50 @@ use std::env; use std::path::PathBuf; -use std::sync::Arc; use deno_cache_dir::DenoDirResolutionError; +use deno_cache_dir::ResolveDenoDirSys; use super::DiskCache; -use crate::factory::CliDenoDirPathProvider; -use crate::sys::CliSys; +use super::DiskCacheSys; + +#[derive(Debug, Clone)] +pub struct DenoDirOptions { + pub maybe_custom_root: Option, +} + +#[sys_traits::auto_impl] +pub trait DenoDirSys: DiskCacheSys + ResolveDenoDirSys + Clone {} + +#[allow(clippy::disallowed_types)] +pub type DenoDirProviderRc = crate::sync::MaybeArc>; /// Lazily creates the deno dir which might be useful in scenarios /// where functionality wants to continue if the DENO_DIR can't be created. -pub struct DenoDirProvider { - deno_dir_path_provider: Arc, - sys: CliSys, - deno_dir: once_cell::sync::OnceCell, +pub struct DenoDirProvider { + options: DenoDirOptions, + sys: TSys, + deno_dir_cell: once_cell::sync::OnceCell>, } -impl DenoDirProvider { - pub fn new( - sys: CliSys, - deno_dir_path_provider: Arc, - ) -> Self { +impl DenoDirProvider { + pub fn new(sys: TSys, options: DenoDirOptions) -> Self { Self { + options, sys, - deno_dir_path_provider, - deno_dir: Default::default(), + deno_dir_cell: Default::default(), } } - pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> { - self.deno_dir.get_or_try_init(|| { - let path = self.deno_dir_path_provider.get_or_create()?; - Ok(DenoDir::new(self.sys.clone(), path.clone())) + pub fn get_or_create( + &self, + ) -> Result<&DenoDir, DenoDirResolutionError> { + self.deno_dir_cell.get_or_try_init(|| { + let path = deno_cache_dir::resolve_deno_dir( + &self.sys, + self.options.maybe_custom_root.clone(), + )?; + Ok(DenoDir::new(self.sys.clone(), path)) }) } } @@ -41,18 +53,19 @@ impl DenoDirProvider { /// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them /// in single directory that can be controlled with `$DENO_DIR` env variable. #[derive(Debug, Clone)] -pub struct DenoDir { +pub struct DenoDir { /// Example: /Users/rld/.deno/ pub root: PathBuf, /// Used by TsCompiler to cache compiler output. - pub gen_cache: DiskCache, + pub gen_cache: DiskCache, } -impl DenoDir { - pub fn new(sys: CliSys, root: PathBuf) -> Self { +impl DenoDir { + pub fn new(sys: TSys, root: PathBuf) -> Self { + #[cfg(not(target_arch = "wasm32"))] assert!(root.is_absolute()); - let gen_path = root.join("gen"); + let gen_path = root.join("gen"); Self { root, gen_cache: DiskCache::new(sys, gen_path), diff --git a/cli/cache/disk_cache.rs b/libs/resolver/cache/disk_cache.rs similarity index 92% rename from cli/cache/disk_cache.rs rename to libs/resolver/cache/disk_cache.rs index a085ef5235..19fd9083fa 100644 --- a/cli/cache/disk_cache.rs +++ b/libs/resolver/cache/disk_cache.rs @@ -9,26 +9,49 @@ use std::str; use deno_cache_dir::url_to_filename; use deno_cache_dir::CACHE_PERM; -use deno_core::url::Host; -use deno_core::url::Url; use deno_path_util::fs::atomic_write_file_with_retries; use sys_traits::FsRead; +use url::Host; +use url::Url; -use crate::sys::CliSys; +#[sys_traits::auto_impl] +pub trait DiskCacheSys: + deno_path_util::fs::AtomicWriteFileWithRetriesSys + FsRead +{ +} #[derive(Debug, Clone)] -pub struct DiskCache { - sys: CliSys, +pub struct DiskCache { + sys: TSys, pub location: PathBuf, } -impl DiskCache { +impl DiskCache { /// `location` must be an absolute path. - pub fn new(sys: CliSys, location: PathBuf) -> Self { + pub fn new(sys: TSys, location: PathBuf) -> Self { + #[cfg(not(target_arch = "wasm32"))] assert!(location.is_absolute()); + Self { sys, location } } + pub fn get_cache_filename_with_extension( + &self, + url: &Url, + extension: &str, + ) -> Option { + let base = self.get_cache_filename(url)?; + + match base.extension() { + None => Some(base.with_extension(extension)), + Some(ext) => { + let original_extension = OsStr::to_str(ext).unwrap(); + let final_extension = format!("{original_extension}.{extension}"); + Some(base.with_extension(final_extension)) + } + } + } + fn get_cache_filename(&self, url: &Url) -> Option { let mut out = PathBuf::new(); @@ -52,13 +75,16 @@ impl DiskCache { } "http" | "https" | "data" | "blob" => out = url_to_filename(url).ok()?, "file" => { - let path = match url.to_file_path() { + let path = match deno_path_util::url_to_file_path(url) { Ok(path) => path, Err(_) => return None, }; let mut path_components = path.components(); - if cfg!(target_os = "windows") { + if sys_traits::impls::is_windows() { + if url.path() == "/" { + return None; // not a valid windows path + } if let Some(Component::Prefix(prefix_component)) = path_components.next() { @@ -97,23 +123,6 @@ impl DiskCache { Some(out) } - pub fn get_cache_filename_with_extension( - &self, - url: &Url, - extension: &str, - ) -> Option { - let base = self.get_cache_filename(url)?; - - match base.extension() { - None => Some(base.with_extension(extension)), - Some(ext) => { - let original_extension = OsStr::to_str(ext).unwrap(); - let final_extension = format!("{original_extension}.{extension}"); - Some(base.with_extension(final_extension)) - } - } - } - pub fn get(&self, filename: &Path) -> std::io::Result> { let path = self.location.join(filename); Ok(self.sys.fs_read(path)?.into_owned()) @@ -268,7 +277,7 @@ mod tests { for test_case in &test_cases { let cache_filename = cache.get_cache_filename(&Url::parse(test_case).unwrap()); - assert_eq!(cache_filename, None); + assert_eq!(cache_filename, None, "Failed for {:?}", test_case); } } } diff --git a/cli/cache/emit.rs b/libs/resolver/cache/emit.rs similarity index 79% rename from cli/cache/emit.rs rename to libs/resolver/cache/emit.rs index 4251135fea..08db2d1321 100644 --- a/cli/cache/emit.rs +++ b/libs/resolver/cache/emit.rs @@ -1,19 +1,28 @@ // Copyright 2018-2025 the Deno authors. MIT license. +use std::borrow::Cow; +use std::hash::Hash; +use std::hash::Hasher; use std::path::PathBuf; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::anyhow; -use deno_core::error::AnyError; -use deno_core::unsync::sync::AtomicFlag; -use deno_lib::version::DENO_VERSION_INFO; +use anyhow::anyhow; +use anyhow::Error as AnyError; +use deno_unsync::sync::AtomicFlag; +use url::Url; use super::DiskCache; +use super::DiskCacheSys; + +#[allow(clippy::disallowed_types)] +pub type EmitCacheRc = crate::sync::MaybeArc>; + +#[sys_traits::auto_impl] +pub trait EmitCacheSys: DiskCacheSys + sys_traits::EnvVar {} /// The cache that stores previously emitted files. #[derive(Debug)] -pub struct EmitCache { - disk_cache: DiskCache, +pub struct EmitCache { + disk_cache: DiskCache, emit_failed_flag: AtomicFlag, file_serializer: EmitFileSerializer, mode: Mode, @@ -25,9 +34,14 @@ enum Mode { Disable, } -impl EmitCache { - pub fn new(disk_cache: DiskCache) -> Self { - let mode = match std::env::var("DENO_EMIT_CACHE_MODE") +impl EmitCache { + pub fn new( + sys: &TSys, + disk_cache: DiskCache, + cache_version: Cow<'static, str>, + ) -> Self { + let mode = match sys + .env_var("DENO_EMIT_CACHE_MODE") .unwrap_or_default() .as_str() { @@ -42,9 +56,7 @@ impl EmitCache { Self { disk_cache, emit_failed_flag: Default::default(), - file_serializer: EmitFileSerializer { - cli_version: DENO_VERSION_INFO.deno, - }, + file_serializer: EmitFileSerializer { cache_version }, mode, } } @@ -59,7 +71,7 @@ impl EmitCache { /// or emits that do not match the source. pub fn get_emit_code( &self, - specifier: &ModuleSpecifier, + specifier: &Url, expected_source_hash: u64, ) -> Option { if matches!(self.mode, Mode::Disable) { @@ -74,12 +86,7 @@ impl EmitCache { } /// Sets the emit code in the cache. - pub fn set_emit_code( - &self, - specifier: &ModuleSpecifier, - source_hash: u64, - code: &[u8], - ) { + pub fn set_emit_code(&self, specifier: &Url, source_hash: u64, code: &[u8]) { if let Err(err) = self.set_emit_code_result(specifier, source_hash, code) { // might error in cases such as a readonly file system log::debug!("Error saving emit data ({}): {}", specifier, err); @@ -90,7 +97,7 @@ impl EmitCache { fn set_emit_code_result( &self, - specifier: &ModuleSpecifier, + specifier: &Url, source_hash: u64, code: &[u8], ) -> Result<(), AnyError> { @@ -108,7 +115,7 @@ impl EmitCache { Ok(()) } - fn get_emit_filename(&self, specifier: &ModuleSpecifier) -> Option { + fn get_emit_filename(&self, specifier: &Url) -> Option { self .disk_cache .get_cache_filename_with_extension(specifier, "js") @@ -119,7 +126,7 @@ const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata="; #[derive(Debug)] struct EmitFileSerializer { - cli_version: &'static str, + cache_version: Cow<'static, str>, } impl EmitFileSerializer { @@ -172,11 +179,11 @@ impl EmitFileSerializer { // it's ok to use an insecure hash here because // if someone can change the emit source then they // can also change the version hash - deno_lib::util::hash::FastInsecureHasher::new_without_deno_version() // use cli_version property instead - .write(bytes) - // emit should not be re-used between cli versions - .write_str(self.cli_version) - .finish() + let mut hasher = twox_hash::XxHash64::default(); + bytes.hash(&mut hasher); + // emit should not be re-used between cli versions + self.cache_version.hash(&mut hasher); + hasher.finish() } } @@ -185,28 +192,29 @@ mod test { use test_util::TempDir; use super::*; - use crate::sys::CliSys; #[test] pub fn emit_cache_general_use() { let temp_dir = TempDir::new(); let disk_cache = - DiskCache::new(CliSys::default(), temp_dir.path().to_path_buf()); + DiskCache::new(sys_traits::impls::RealSys, temp_dir.path().to_path_buf()); let cache = EmitCache { disk_cache: disk_cache.clone(), file_serializer: EmitFileSerializer { - cli_version: "1.0.0", + cache_version: "1.0.0".into(), }, emit_failed_flag: Default::default(), mode: Mode::Normal, }; - let specifier1 = - ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts")) - .unwrap(); - let specifier2 = - ModuleSpecifier::from_file_path(temp_dir.path().join("file2.ts")) - .unwrap(); + let specifier1 = deno_path_util::url_from_file_path( + temp_dir.path().join("file1.ts").as_path(), + ) + .unwrap(); + let specifier2 = deno_path_util::url_from_file_path( + temp_dir.path().join("file2.ts").as_path(), + ) + .unwrap(); assert_eq!(cache.get_emit_code(&specifier1, 1), None); let emit_code1 = "text1".to_string(); let emit_code2 = "text2".to_string(); @@ -225,7 +233,7 @@ mod test { let cache = EmitCache { disk_cache: disk_cache.clone(), file_serializer: EmitFileSerializer { - cli_version: "2.0.0", + cache_version: "2.0.0".into(), }, emit_failed_flag: Default::default(), mode: Mode::Normal, @@ -237,7 +245,7 @@ mod test { let cache = EmitCache { disk_cache, file_serializer: EmitFileSerializer { - cli_version: "2.0.0", + cache_version: "2.0.0".into(), }, emit_failed_flag: Default::default(), mode: Mode::Normal, diff --git a/libs/resolver/cache/mod.rs b/libs/resolver/cache/mod.rs new file mode 100644 index 0000000000..a48c38e1ee --- /dev/null +++ b/libs/resolver/cache/mod.rs @@ -0,0 +1,24 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +mod deno_dir; +mod disk_cache; +mod emit; +#[cfg(feature = "deno_ast")] +mod parsed_source; + +pub use deno_dir::DenoDir; +pub use deno_dir::DenoDirOptions; +pub use deno_dir::DenoDirProvider; +pub use deno_dir::DenoDirProviderRc; +pub use deno_dir::DenoDirSys; +pub use disk_cache::DiskCache; +pub use disk_cache::DiskCacheSys; +pub use emit::EmitCache; +pub use emit::EmitCacheRc; +pub use emit::EmitCacheSys; +#[cfg(feature = "deno_ast")] +pub use parsed_source::LazyGraphSourceParser; +#[cfg(feature = "deno_ast")] +pub use parsed_source::ParsedSourceCache; +#[cfg(feature = "deno_ast")] +pub use parsed_source::ParsedSourceCacheRc; diff --git a/cli/cache/parsed_source.rs b/libs/resolver/cache/parsed_source.rs similarity index 65% rename from cli/cache/parsed_source.rs rename to libs/resolver/cache/parsed_source.rs index 79f6b1cdba..96e523780a 100644 --- a/cli/cache/parsed_source.rs +++ b/libs/resolver/cache/parsed_source.rs @@ -1,17 +1,12 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::collections::HashMap; -use std::sync::Arc; - -use deno_ast::MediaType; -use deno_ast::ModuleSpecifier; use deno_ast::ParsedSource; -use deno_core::parking_lot::Mutex; use deno_graph::ast::CapturingEsParser; use deno_graph::ast::DefaultEsParser; use deno_graph::ast::EsParser; -use deno_graph::ast::ParseOptions; use deno_graph::ast::ParsedSourceStore; +use deno_media_type::MediaType; +use url::Url; /// Lazily parses JS/TS sources from a `deno_graph::ModuleGraph` given /// a `ParsedSourceCache`. Note that deno_graph doesn't necessarily cause @@ -34,8 +29,8 @@ impl<'a> LazyGraphSourceParser<'a> { #[allow(clippy::result_large_err)] pub fn get_or_parse_source( &self, - module_specifier: &ModuleSpecifier, - ) -> Result, deno_ast::ParseDiagnostic> { + module_specifier: &Url, + ) -> Result, deno_ast::ParseDiagnostic> { let Some(deno_graph::Module::Js(module)) = self.graph.get(module_specifier) else { return Ok(None); @@ -47,9 +42,15 @@ impl<'a> LazyGraphSourceParser<'a> { } } +#[allow(clippy::disallowed_types)] // ok because we always store source text as Arc +type ArcStr = std::sync::Arc; + +#[allow(clippy::disallowed_types)] +pub type ParsedSourceCacheRc = crate::sync::MaybeArc; + #[derive(Debug, Default)] pub struct ParsedSourceCache { - sources: Mutex>, + sources: crate::sync::MaybeDashMap, } impl ParsedSourceCache { @@ -60,7 +61,7 @@ impl ParsedSourceCache { ) -> Result { let parser = self.as_capturing_parser(); // this will conditionally parse because it's using a CapturingEsParser - parser.parse_program(ParseOptions { + parser.parse_program(deno_graph::ast::ParseOptions { specifier: &module.specifier, source: module.source.text.clone(), media_type: module.media_type, @@ -68,11 +69,11 @@ impl ParsedSourceCache { }) } - #[allow(clippy::result_large_err)] + #[allow(clippy::result_large_err, clippy::disallowed_types)] pub fn remove_or_parse_module( &self, - specifier: &ModuleSpecifier, - source: Arc, + specifier: &Url, + source: ArcStr, media_type: MediaType, ) -> Result { if let Some(parsed_source) = self.remove_parsed_source(specifier) { @@ -84,7 +85,7 @@ impl ParsedSourceCache { return Ok(parsed_source); } } - let options = ParseOptions { + let options = deno_graph::ast::ParseOptions { specifier, source, media_type, @@ -94,13 +95,13 @@ impl ParsedSourceCache { } /// Frees the parsed source from memory. - pub fn free(&self, specifier: &ModuleSpecifier) { - self.sources.lock().remove(specifier); + pub fn free(&self, specifier: &Url) { + self.sources.remove(specifier); } /// Fress all parsed sources from memory. pub fn free_all(&self) { - self.sources.lock().clear(); + self.sources.clear(); } /// Creates a parser that will reuse a ParsedSource from the store @@ -109,9 +110,9 @@ impl ParsedSourceCache { CapturingEsParser::new(None, self) } - #[cfg(test)] + #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { - self.sources.lock().len() + self.sources.len() } } @@ -120,43 +121,37 @@ impl ParsedSourceCache { /// and in LSP settings the concurrency will be enforced /// at a higher level to ensure this will have the latest /// parsed source. -impl deno_graph::ast::ParsedSourceStore for ParsedSourceCache { +impl ParsedSourceStore for ParsedSourceCache { fn set_parsed_source( &self, - specifier: ModuleSpecifier, + specifier: Url, parsed_source: ParsedSource, ) -> Option { - self.sources.lock().insert(specifier, parsed_source) + self.sources.insert(specifier, parsed_source) } - fn get_parsed_source( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - self.sources.lock().get(specifier).cloned() + fn get_parsed_source(&self, specifier: &Url) -> Option { + self.sources.get(specifier).map(|p| p.clone()) } - fn remove_parsed_source( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - self.sources.lock().remove(specifier) + fn remove_parsed_source(&self, specifier: &Url) -> Option { + self.sources.remove(specifier).map(|(_, p)| p) } fn get_scope_analysis_parsed_source( &self, - specifier: &ModuleSpecifier, + specifier: &Url, ) -> Option { - let mut sources = self.sources.lock(); - let parsed_source = sources.get(specifier)?; - if parsed_source.has_scope_analysis() { - Some(parsed_source.clone()) - } else { - // upgrade to have scope analysis - let parsed_source = sources.remove(specifier).unwrap(); - let parsed_source = parsed_source.into_with_scope_analysis(); - sources.insert(specifier.clone(), parsed_source.clone()); - Some(parsed_source) + { + let parsed_source = self.sources.get(specifier)?; + if parsed_source.has_scope_analysis() { + return Some(parsed_source.clone()); + } } + // upgrade to have scope analysis + let (specifier, parsed_source) = self.sources.remove(specifier)?; + let parsed_source = parsed_source.into_with_scope_analysis(); + self.sources.insert(specifier, parsed_source.clone()); + Some(parsed_source) } } diff --git a/cli/emit.rs b/libs/resolver/emit.rs similarity index 75% rename from cli/emit.rs rename to libs/resolver/emit.rs index 8e07705f36..443c62df89 100644 --- a/cli/emit.rs +++ b/libs/resolver/emit.rs @@ -1,7 +1,9 @@ // Copyright 2018-2025 the Deno authors. MIT license. -use std::sync::Arc; +use std::hash::Hash; +use std::hash::Hasher; +use anyhow::Error as AnyError; use deno_ast::EmittedSourceText; use deno_ast::ModuleKind; use deno_ast::SourceMapOption; @@ -10,38 +12,51 @@ use deno_ast::SourceRanged; use deno_ast::SourceRangedForSpanned; use deno_ast::TranspileModuleOptions; use deno_ast::TranspileResult; -use deno_core::error::AnyError; -use deno_core::error::CoreError; -use deno_core::futures::stream::FuturesUnordered; -use deno_core::futures::FutureExt; -use deno_core::futures::StreamExt; -use deno_core::ModuleSpecifier; use deno_error::JsErrorBox; use deno_graph::MediaType; use deno_graph::Module; use deno_graph::ModuleGraph; -use deno_lib::util::hash::FastInsecureHasher; -use deno_resolver::deno_json::CompilerOptionsResolver; -use deno_resolver::deno_json::TranspileAndEmitOptions; +use futures::stream::FuturesUnordered; +use futures::FutureExt; +use futures::StreamExt; +use node_resolver::InNpmPackageChecker; +use url::Url; -use crate::cache::EmitCache; +use crate::cache::EmitCacheRc; +use crate::cache::EmitCacheSys; use crate::cache::ParsedSourceCache; -use crate::resolver::CliCjsTracker; +use crate::cache::ParsedSourceCacheRc; +use crate::cjs::CjsTrackerRc; +use crate::deno_json::CompilerOptionsResolverRc; +use crate::deno_json::TranspileAndEmitOptions; + +#[allow(clippy::disallowed_types)] // ok because we always store source text as Arc +type ArcStr = std::sync::Arc; + +#[allow(clippy::disallowed_types)] +pub type EmitterRc = + crate::sync::MaybeArc>; + +#[sys_traits::auto_impl] +pub trait EmitterSys: EmitCacheSys {} #[derive(Debug)] -pub struct Emitter { - cjs_tracker: Arc, - emit_cache: Arc, - parsed_source_cache: Arc, - compiler_options_resolver: Arc, +pub struct Emitter +{ + cjs_tracker: CjsTrackerRc, + emit_cache: EmitCacheRc, + parsed_source_cache: ParsedSourceCacheRc, + compiler_options_resolver: CompilerOptionsResolverRc, } -impl Emitter { +impl + Emitter +{ pub fn new( - cjs_tracker: Arc, - emit_cache: Arc, - parsed_source_cache: Arc, - compiler_options_resolver: Arc, + cjs_tracker: CjsTrackerRc, + emit_cache: EmitCacheRc, + parsed_source_cache: ParsedSourceCacheRc, + compiler_options_resolver: CompilerOptionsResolverRc, ) -> Self { Self { cjs_tracker, @@ -91,7 +106,7 @@ impl Emitter { /// Gets a cached emit if the source matches the hash found in the cache. pub fn maybe_cached_emit( &self, - specifier: &ModuleSpecifier, + specifier: &Url, module_kind: deno_ast::ModuleKind, source: &str, ) -> Result, AnyError> { @@ -106,10 +121,10 @@ impl Emitter { pub async fn emit_parsed_source( &self, - specifier: &ModuleSpecifier, + specifier: &Url, media_type: MediaType, module_kind: ModuleKind, - source: &Arc, + source: &ArcStr, ) -> Result { let transpile_and_emit_options = self .compiler_options_resolver @@ -127,11 +142,12 @@ impl Emitter { PreEmitResult::NotCached { source_hash } => { let parsed_source_cache = self.parsed_source_cache.clone(); let transpile_and_emit_options = transpile_and_emit_options.clone(); - let transpiled_source = deno_core::unsync::spawn_blocking({ + #[cfg(feature = "sync")] + let transpiled_source = crate::rt::spawn_blocking({ let specifier = specifier.clone(); let source = source.clone(); move || { - EmitParsedSourceHelper::transpile( + transpile( &parsed_source_cache, &specifier, media_type, @@ -145,6 +161,17 @@ impl Emitter { }) .await .unwrap()?; + #[cfg(not(feature = "sync"))] + let transpiled_source = transpile( + &parsed_source_cache, + &specifier, + media_type, + module_kind, + source.clone(), + &transpile_and_emit_options.transpile, + &transpile_and_emit_options.emit, + )? + .text; helper.post_emit_parsed_source( specifier, &transpiled_source, @@ -158,10 +185,10 @@ impl Emitter { #[allow(clippy::result_large_err)] pub fn emit_parsed_source_sync( &self, - specifier: &ModuleSpecifier, + specifier: &Url, media_type: MediaType, module_kind: deno_ast::ModuleKind, - source: &Arc, + source: &ArcStr, ) -> Result { let transpile_and_emit_options = self .compiler_options_resolver @@ -177,7 +204,7 @@ impl Emitter { ) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { - let transpiled_source = EmitParsedSourceHelper::transpile( + let transpiled_source = transpile( &self.parsed_source_cache, specifier, media_type, @@ -199,10 +226,10 @@ impl Emitter { pub fn emit_parsed_source_for_deno_compile( &self, - specifier: &ModuleSpecifier, + specifier: &Url, media_type: MediaType, module_kind: deno_ast::ModuleKind, - source: &Arc, + source: &ArcStr, ) -> Result<(String, String), AnyError> { let transpile_and_emit_options = self .compiler_options_resolver @@ -214,7 +241,7 @@ impl Emitter { // strip off the path to have more deterministic builds as we don't care // about the source name because we manually provide the source map to v8 emit_options.source_map_base = Some(deno_path_util::url_parent(specifier)); - let source = EmitParsedSourceHelper::transpile( + let source = transpile( &self.parsed_source_cache, specifier, media_type, @@ -227,22 +254,19 @@ impl Emitter { } /// Expects a file URL, panics otherwise. - pub async fn load_and_emit_for_hmr( + pub fn emit_for_hmr( &self, - specifier: &ModuleSpecifier, - ) -> Result { + specifier: &Url, + source_code: String, + ) -> Result { let media_type = MediaType::from_specifier(specifier); - let source_code = tokio::fs::read_to_string( - ModuleSpecifier::to_file_path(specifier).unwrap(), - ) - .await?; match media_type { MediaType::TypeScript | MediaType::Mts | MediaType::Cts | MediaType::Jsx | MediaType::Tsx => { - let source_arc: Arc = source_code.into(); + let source_arc: ArcStr = source_code.into(); let parsed_source = self .parsed_source_cache .remove_or_parse_module(specifier, source_arc, media_type) @@ -306,11 +330,11 @@ impl Emitter { transpile_and_emit: &TranspileAndEmitOptions, source_text: &str, ) -> u64 { - FastInsecureHasher::new_without_deno_version() // stored in the transpile_and_emit_options_hash - .write_str(source_text) - .write_u64(transpile_and_emit.pre_computed_hash) - .write_hashable(module_kind) - .finish() + let mut hasher = twox_hash::XxHash64::default(); + source_text.hash(&mut hasher); + transpile_and_emit.pre_computed_hash.hash(&mut hasher); + module_kind.hash(&mut hasher); + hasher.finish() } } @@ -338,15 +362,21 @@ pub enum EmitParsedSourceHelperError { } /// Helper to share code between async and sync emit_parsed_source methods. -struct EmitParsedSourceHelper<'a>(&'a Emitter); +struct EmitParsedSourceHelper< + 'a, + TInNpmPackageChecker: InNpmPackageChecker, + TSys: EmitterSys, +>(&'a Emitter); -impl EmitParsedSourceHelper<'_> { +impl + EmitParsedSourceHelper<'_, TInNpmPackageChecker, TSys> +{ pub fn pre_emit_parsed_source( &self, - specifier: &ModuleSpecifier, + specifier: &Url, module_kind: deno_ast::ModuleKind, transpile_and_emit_options: &TranspileAndEmitOptions, - source: &Arc, + source: &ArcStr, ) -> PreEmitResult { let source_hash = self @@ -362,41 +392,9 @@ impl EmitParsedSourceHelper<'_> { } } - #[allow(clippy::result_large_err)] - pub fn transpile( - parsed_source_cache: &ParsedSourceCache, - specifier: &ModuleSpecifier, - media_type: MediaType, - module_kind: deno_ast::ModuleKind, - source: Arc, - transpile_options: &deno_ast::TranspileOptions, - emit_options: &deno_ast::EmitOptions, - ) -> Result { - // nothing else needs the parsed source at this point, so remove from - // the cache in order to not transpile owned - let parsed_source = parsed_source_cache - .remove_or_parse_module(specifier, source, media_type)?; - ensure_no_import_assertion(&parsed_source)?; - let transpile_result = parsed_source.transpile( - transpile_options, - &TranspileModuleOptions { - module_kind: Some(module_kind), - }, - emit_options, - )?; - let transpiled_source = match transpile_result { - TranspileResult::Owned(source) => source, - TranspileResult::Cloned(source) => { - debug_assert!(false, "Transpile owned failed."); - source - } - }; - Ok(transpiled_source) - } - pub fn post_emit_parsed_source( &self, - specifier: &ModuleSpecifier, + specifier: &Url, transpiled_source: &str, source_hash: u64, ) { @@ -408,6 +406,38 @@ impl EmitParsedSourceHelper<'_> { } } +#[allow(clippy::result_large_err)] +fn transpile( + parsed_source_cache: &ParsedSourceCache, + specifier: &Url, + media_type: MediaType, + module_kind: deno_ast::ModuleKind, + source: ArcStr, + transpile_options: &deno_ast::TranspileOptions, + emit_options: &deno_ast::EmitOptions, +) -> Result { + // nothing else needs the parsed source at this point, so remove from + // the cache in order to not transpile owned + let parsed_source = parsed_source_cache + .remove_or_parse_module(specifier, source, media_type)?; + ensure_no_import_assertion(&parsed_source)?; + let transpile_result = parsed_source.transpile( + transpile_options, + &TranspileModuleOptions { + module_kind: Some(module_kind), + }, + emit_options, + )?; + let transpiled_source = match transpile_result { + TranspileResult::Owned(source) => source, + TranspileResult::Cloned(source) => { + debug_assert!(false, "Transpile owned failed."); + source + } + }; + Ok(transpiled_source) +} + // todo(dsherret): this is a temporary measure until we have swc erroring for this fn ensure_no_import_assertion( parsed_source: &deno_ast::ParsedSource, diff --git a/libs/resolver/factory.rs b/libs/resolver/factory.rs index c06e6bde72..02e4f1e746 100644 --- a/libs/resolver/factory.rs +++ b/libs/resolver/factory.rs @@ -31,13 +31,16 @@ use node_resolver::NodeResolverOptions; use node_resolver::NodeResolverRc; use node_resolver::PackageJsonResolver; use node_resolver::PackageJsonResolverRc; -use sys_traits::EnvCacheDir; -use sys_traits::EnvCurrentDir; -use sys_traits::EnvHomeDir; -use sys_traits::EnvVar; use thiserror::Error; use url::Url; +use crate::cache::DenoDir; +use crate::cache::DenoDirOptions; +use crate::cache::DenoDirProvider; +use crate::cache::DenoDirProviderRc; +use crate::cache::DenoDirSys; +use crate::cache::EmitCache; +use crate::cache::EmitCacheRc; use crate::cjs::CjsTracker; use crate::cjs::CjsTrackerRc; use crate::cjs::IsCjsResolutionMode; @@ -163,49 +166,6 @@ pub trait SpecifiedImportMapProvider: ) -> Result, anyhow::Error>; } -#[derive(Debug, Clone)] -pub struct DenoDirPathProviderOptions { - pub maybe_custom_root: Option, -} - -#[allow(clippy::disallowed_types)] -pub type DenoDirPathProviderRc = - crate::sync::MaybeArc>; - -#[sys_traits::auto_impl] -pub trait DenoDirPathProviderSys: - EnvCacheDir + EnvHomeDir + EnvVar + EnvCurrentDir -{ -} - -/// Lazily creates the deno dir which might be useful in scenarios -/// where functionality wants to continue if the DENO_DIR can't be created. -#[derive(Debug)] -pub struct DenoDirPathProvider { - sys: TSys, - options: DenoDirPathProviderOptions, - deno_dir: Deferred, -} - -impl DenoDirPathProvider { - pub fn new(sys: TSys, options: DenoDirPathProviderOptions) -> Self { - Self { - sys, - options, - deno_dir: Default::default(), - } - } - - pub fn get_or_create(&self) -> Result<&PathBuf, DenoDirResolutionError> { - self.deno_dir.get_or_try_init(|| { - deno_cache_dir::resolve_deno_dir( - &self.sys, - self.options.maybe_custom_root.clone(), - ) - }) - } -} - #[derive(Debug)] pub struct NpmProcessStateOptions { pub node_modules_dir: Option>, @@ -213,15 +173,18 @@ pub struct NpmProcessStateOptions { } #[derive(Debug, Default)] -pub struct WorkspaceFactoryOptions { +pub struct WorkspaceFactoryOptions { pub additional_config_file_names: &'static [&'static str], pub config_discovery: ConfigDiscoveryOption, - pub deno_dir_path_provider: Option>, pub is_package_manager_subcommand: bool, + /// Version to use for the emit cache. This is something that + /// should change when the version of the underlying emit changes. + pub emit_cache_version: Cow<'static, str>, pub frozen_lockfile: Option, pub lock_arg: Option, /// Whether to skip writing to the lockfile. pub lockfile_skip_write: bool, + pub maybe_custom_deno_dir_root: Option, pub node_modules_dir: Option, pub no_lock: bool, pub no_npm: bool, @@ -238,7 +201,7 @@ pub type WorkspaceFactoryRc = #[sys_traits::auto_impl] pub trait WorkspaceFactorySys: - DenoDirPathProviderSys + DenoDirSys + crate::lockfile::LockfileSys + crate::npm::NpmResolverSys + deno_cache_dir::GlobalHttpCacheSys @@ -248,7 +211,8 @@ pub trait WorkspaceFactorySys: pub struct WorkspaceFactory { sys: TSys, - deno_dir_path: DenoDirPathProviderRc, + deno_dir_provider: Deferred>, + emit_cache: Deferred>, global_http_cache: Deferred>, http_cache: Deferred>, jsr_url: Deferred, @@ -263,27 +227,19 @@ pub struct WorkspaceFactory { Deferred>, workspace_npm_link_packages: Deferred, initial_cwd: PathBuf, - options: WorkspaceFactoryOptions, + options: WorkspaceFactoryOptions, } impl WorkspaceFactory { pub fn new( sys: TSys, initial_cwd: PathBuf, - mut options: WorkspaceFactoryOptions, + options: WorkspaceFactoryOptions, ) -> Self { Self { - deno_dir_path: options.deno_dir_path_provider.take().unwrap_or_else( - || { - new_rc(DenoDirPathProvider::new( - sys.clone(), - DenoDirPathProviderOptions { - maybe_custom_root: None, - }, - )) - }, - ), sys, + deno_dir_provider: Default::default(), + emit_cache: Default::default(), global_http_cache: Default::default(), http_cache: Default::default(), jsr_url: Default::default(), @@ -308,6 +264,27 @@ impl WorkspaceFactory { self.workspace_directory = Deferred::from(workspace_directory); } + pub fn deno_dir_provider(&self) -> &DenoDirProviderRc { + self.deno_dir_provider.get_or_init(|| { + new_rc(DenoDirProvider::new( + self.sys.clone(), + DenoDirOptions { + maybe_custom_root: self.options.maybe_custom_deno_dir_root.clone(), + }, + )) + }) + } + + pub fn emit_cache(&self) -> Result<&EmitCacheRc, anyhow::Error> { + self.emit_cache.get_or_try_init(|| { + Ok(new_rc(EmitCache::new( + &self.sys, + self.deno_dir()?.gen_cache.clone(), + self.options.emit_cache_version.clone(), + ))) + }) + } + pub fn jsr_url(&self) -> &Url { self.jsr_url.get_or_init(|| resolve_jsr_url(&self.sys)) } @@ -348,7 +325,8 @@ impl WorkspaceFactory { let workspace = &self.workspace_directory()?.workspace; if let Some(pkg_json) = workspace.root_pkg_json() { - if let Ok(deno_dir) = self.deno_dir_path() { + if let Ok(deno_dir) = self.deno_dir() { + let deno_dir = &deno_dir.root; // `deno_dir` can be symlink in macOS or on the CI if let Ok(deno_dir) = canonicalize_path_maybe_not_exists(&self.sys, deno_dir) @@ -431,15 +409,15 @@ impl WorkspaceFactory { .map(|p| p.as_deref()) } - pub fn deno_dir_path(&self) -> Result<&PathBuf, DenoDirResolutionError> { - self.deno_dir_path.get_or_create() + pub fn deno_dir(&self) -> Result<&DenoDir, DenoDirResolutionError> { + self.deno_dir_provider().get_or_create() } pub fn global_http_cache( &self, ) -> Result<&GlobalHttpCacheRc, DenoDirResolutionError> { self.global_http_cache.get_or_try_init(|| { - let global_cache_dir = self.deno_dir_path()?.join("remote"); + let global_cache_dir = self.deno_dir()?.remote_folder_path(); let global_http_cache = new_rc(deno_cache_dir::GlobalHttpCache::new( self.sys.clone(), global_cache_dir, @@ -514,7 +492,7 @@ impl WorkspaceFactory { &self, ) -> Result<&NpmCacheDirRc, NpmCacheDirCreateError> { self.npm_cache_dir.get_or_try_init(|| { - let npm_cache_dir = self.deno_dir_path()?.join("npm"); + let npm_cache_dir = self.deno_dir()?.npm_folder_path(); Ok(new_rc(NpmCacheDir::new( &self.sys, npm_cache_dir, @@ -683,6 +661,8 @@ pub struct ResolverFactory { #[cfg(feature = "graph")] deno_resolver: async_once_cell::OnceCell>, + #[cfg(feature = "deno_ast")] + emitter: Deferred>, #[cfg(feature = "graph")] found_package_json_dep_flag: crate::graph::FoundPackageJsonDepFlagRc, in_npm_package_checker: Deferred, @@ -704,6 +684,8 @@ pub struct ResolverFactory { >, npm_resolver: Deferred>, npm_resolution: NpmResolutionCellRc, + #[cfg(feature = "deno_ast")] + parsed_source_cache: crate::cache::ParsedSourceCacheRc, pkg_json_resolver: Deferred>, raw_deno_resolver: async_once_cell::OnceCell>, workspace_factory: WorkspaceFactoryRc, @@ -725,6 +707,8 @@ impl ResolverFactory { raw_deno_resolver: Default::default(), #[cfg(feature = "graph")] deno_resolver: Default::default(), + #[cfg(feature = "deno_ast")] + emitter: Default::default(), #[cfg(feature = "graph")] found_package_json_dep_flag: Default::default(), in_npm_package_checker: Default::default(), @@ -732,6 +716,8 @@ impl ResolverFactory { npm_req_resolver: Default::default(), npm_resolution: Default::default(), npm_resolver: Default::default(), + #[cfg(feature = "deno_ast")] + parsed_source_cache: Default::default(), pkg_json_resolver: Default::default(), workspace_factory, workspace_resolver: Default::default(), @@ -798,6 +784,23 @@ impl ResolverFactory { }) } + #[cfg(feature = "deno_ast")] + pub fn emitter( + &self, + ) -> Result< + &crate::emit::EmitterRc, + anyhow::Error, + > { + self.emitter.get_or_try_init(|| { + Ok(new_rc(crate::emit::Emitter::new( + self.cjs_tracker()?.clone(), + self.workspace_factory.emit_cache()?.clone(), + self.parsed_source_cache().clone(), + self.compiler_options_resolver()?.clone(), + ))) + }) + } + #[cfg(feature = "graph")] pub fn found_package_json_dep_flag( &self, @@ -936,6 +939,11 @@ impl ResolverFactory { }) } + #[cfg(feature = "deno_ast")] + pub fn parsed_source_cache(&self) -> &crate::cache::ParsedSourceCacheRc { + &self.parsed_source_cache + } + pub fn workspace_factory(&self) -> &WorkspaceFactoryRc { &self.workspace_factory } diff --git a/libs/resolver/lib.rs b/libs/resolver/lib.rs index c9a9cff5ba..c81a399db1 100644 --- a/libs/resolver/lib.rs +++ b/libs/resolver/lib.rs @@ -41,10 +41,13 @@ use crate::workspace::MappedResolutionError; use crate::workspace::WorkspaceResolvePkgJsonFolderError; use crate::workspace::WorkspaceResolver; +pub mod cache; pub mod cjs; pub mod collections; pub mod deno_json; pub mod display; +#[cfg(feature = "deno_ast")] +pub mod emit; pub mod factory; #[cfg(feature = "graph")] pub mod file_fetcher; @@ -54,6 +57,8 @@ pub mod import_map; pub mod lockfile; pub mod npm; pub mod npmrc; +#[cfg(all(feature = "deno_ast", feature = "sync"))] +mod rt; mod sync; pub mod workspace; diff --git a/libs/resolver/rt.rs b/libs/resolver/rt.rs new file mode 100644 index 0000000000..83fbbb1da2 --- /dev/null +++ b/libs/resolver/rt.rs @@ -0,0 +1,24 @@ +// Copyright 2018-2025 the Deno authors. MIT license. + +#[cfg(not(target_arch = "wasm32"))] +use deno_unsync::JoinHandle; +#[cfg(target_arch = "wasm32")] +pub type JoinHandle = + std::future::Ready>; + +pub fn spawn_blocking< + F: (FnOnce() -> R) + Send + 'static, + R: Send + 'static, +>( + f: F, +) -> JoinHandle { + #[cfg(target_arch = "wasm32")] + { + let result = f(); + std::future::ready(Ok(result)) + } + #[cfg(not(target_arch = "wasm32"))] + { + deno_unsync::spawn_blocking(f) + } +} diff --git a/libs/resolver/sync.rs b/libs/resolver/sync.rs index fb2de4c7fb..88be8b3274 100644 --- a/libs/resolver/sync.rs +++ b/libs/resolver/sync.rs @@ -58,6 +58,19 @@ mod inner { let mut inner = self.0.borrow_mut(); inner.insert(key, value) } + + pub fn clear(&self) { + self.0.borrow_mut().clear(); + } + + pub fn remove(&self, key: &K) -> Option<(K, V)> { + self.0.borrow_mut().remove_entry(key) + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + self.0.borrow().len() + } } // Wrapper struct that exposes a subset of `DashMap` API.