refactor(emitter): ability to not transpile and specify a source map base (#29996)
Some checks failed
ci / pre-build (push) Has been cancelled
ci / build libs (push) Has been cancelled
ci / test debug linux-aarch64 (push) Has been cancelled
ci / test release linux-aarch64 (push) Has been cancelled
ci / test debug macos-aarch64 (push) Has been cancelled
ci / test release macos-aarch64 (push) Has been cancelled
ci / bench release linux-x86_64 (push) Has been cancelled
ci / lint debug linux-x86_64 (push) Has been cancelled
ci / lint debug macos-x86_64 (push) Has been cancelled
ci / lint debug windows-x86_64 (push) Has been cancelled
ci / test debug linux-x86_64 (push) Has been cancelled
ci / test release linux-x86_64 (push) Has been cancelled
ci / test debug macos-x86_64 (push) Has been cancelled
ci / test release macos-x86_64 (push) Has been cancelled
ci / test debug windows-x86_64 (push) Has been cancelled
ci / test release windows-x86_64 (push) Has been cancelled
ci / publish canary (push) Has been cancelled

This commit is contained in:
David Sherret 2025-07-04 12:51:17 -04:00 committed by GitHub
parent 51c43ce8b4
commit 90058d6732
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 205 additions and 84 deletions

View file

@ -1108,6 +1108,11 @@ impl CliFactory {
Ok(Arc::new(CliResolverFactory::new(
self.workspace_factory()?.clone(),
ResolverFactoryOptions {
compiler_options_overrides: CompilerOptionsOverrides {
no_transpile: false,
source_map_base: None,
preserve_jsx: false,
},
is_cjs_resolution_mode: if options.is_node_main()
|| options.unstable_detect_cjs()
{
@ -1208,9 +1213,6 @@ fn new_workspace_factory_options(
} else {
&[]
},
compiler_options_overrides: CompilerOptionsOverrides {
preserve_jsx: false,
},
config_discovery: match &flags.config_flag {
ConfigFlag::Discover => {
if let Some(start_paths) = flags.config_path_args(initial_cwd) {

View file

@ -1474,7 +1474,6 @@ impl ConfigData {
member_dir.dir_path(),
WorkspaceFactoryOptions {
additional_config_file_names: &[],
compiler_options_overrides: Default::default(),
config_discovery: ConfigDiscoveryOption::DiscoverCwd,
maybe_custom_deno_dir_root: None,
is_package_manager_subcommand: false,
@ -1497,6 +1496,7 @@ impl ConfigData {
ResolverFactoryOptions {
// these default options are fine because we don't use this for
// anything other than resolving the lockfile at the moment
compiler_options_overrides: Default::default(),
is_cjs_resolution_mode: Default::default(),
npm_system_info: Default::default(),
node_code_translator_mode: Default::default(),

View file

@ -19,6 +19,7 @@ use boxed_error::Boxed;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_cache_dir::file_fetcher::FetchLocalOptions;
use deno_core::FastString;
use deno_core::ModuleLoader;
use deno_core::ModuleSource;
use deno_core::ModuleSourceCode;
@ -1438,7 +1439,7 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
fn load_text_file_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, JsErrorBox> {
) -> Result<FastString, JsErrorBox> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text = self
@ -1453,9 +1454,9 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
specifier,
}));
}
self
let text = self
.emitter
.emit_parsed_source_sync(
.maybe_emit_source_sync(
&specifier,
media_type,
// this is probably not super accurate due to require esm, but probably ok.
@ -1464,10 +1465,13 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
ModuleKind::Cjs,
&text.into(),
)
.map(Cow::Owned)
.map_err(JsErrorBox::from_err)
.map_err(JsErrorBox::from_err)?;
Ok(text.into())
} else {
Ok(text)
Ok(match text {
Cow::Borrowed(s) => FastString::from_static(s),
Cow::Owned(s) => s.into(),
})
}
}
@ -1596,7 +1600,7 @@ mod tests {
let source = "const a = 'hello';";
let parsed_source_cache = Arc::new(ParsedSourceCache::default());
let parsed_source = parsed_source_cache
.remove_or_parse_module(&specifier, source.into(), MediaType::JavaScript)
.remove_or_parse_module(&specifier, MediaType::JavaScript, source.into())
.unwrap();
parsed_source_cache.set_parsed_source(specifier, parsed_source);

View file

@ -608,7 +608,7 @@ impl NodeRequireLoader for EmbeddedModuleLoader {
fn load_text_file_lossy(
&self,
path: &std::path::Path,
) -> Result<Cow<'static, str>, JsErrorBox> {
) -> Result<FastString, JsErrorBox> {
let file_entry = self
.shared
.vfs
@ -621,7 +621,10 @@ impl NodeRequireLoader for EmbeddedModuleLoader {
file_entry.transpiled_offset.unwrap_or(file_entry.offset),
)
.map_err(JsErrorBox::from_err)?;
Ok(from_utf8_lossy_cow(file_bytes))
Ok(match from_utf8_lossy_cow(file_bytes) {
Cow::Borrowed(s) => FastString::from_static(s),
Cow::Owned(s) => s.into(),
})
}
fn is_maybe_cjs(&self, specifier: &Url) -> Result<bool, ClosestPkgJsonError> {

View file

@ -464,7 +464,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
_ => ModuleKind::Esm,
};
let (source, source_map) =
self.emitter.emit_parsed_source_for_deno_compile(
self.emitter.emit_source_for_deno_compile(
&m.specifier,
m.media_type,
module_kind,

View file

@ -8,6 +8,7 @@ use std::collections::HashSet;
use std::path::Path;
use std::path::PathBuf;
use deno_core::FastString;
use deno_core::OpState;
use deno_core::op2;
use deno_core::url::Url;
@ -164,10 +165,8 @@ pub trait NodeRequireLoader {
path: &'a Path,
) -> Result<Cow<'a, Path>, JsErrorBox>;
fn load_text_file_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, JsErrorBox>;
fn load_text_file_lossy(&self, path: &Path)
-> Result<FastString, JsErrorBox>;
/// Get if the module kind is maybe CJS and loading should determine
/// if its CJS or ESM.

View file

@ -558,10 +558,6 @@ where
let loader = state.borrow::<NodeRequireLoaderRc>();
loader
.load_text_file_lossy(&file_path)
.map(|s| match s {
Cow::Borrowed(s) => FastString::from_static(s),
Cow::Owned(s) => s.into(),
})
.map_err(|e| RequireErrorKind::ReadModule(e).into_box())
}

View file

@ -1156,6 +1156,7 @@ impl ConfigFile {
sys: &impl FsRead,
config_path: &Path,
) -> Result<Self, ConfigFileReadError> {
#[cfg(not(target_arch = "wasm32"))]
debug_assert!(config_path.is_absolute());
let specifier = url_from_file_path(config_path).map_err(|_| {
ConfigFileReadErrorKind::PathToUrl(config_path.to_path_buf()).into_box()

View file

@ -58,13 +58,27 @@ impl ParsedSourceCache {
pub fn get_parsed_source_from_js_module(
&self,
module: &deno_graph::JsModule,
) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
self.get_matching_parsed_source(
&module.specifier,
module.media_type,
module.source.text.clone(),
)
}
#[allow(clippy::result_large_err)]
pub fn get_matching_parsed_source(
&self,
specifier: &Url,
media_type: MediaType,
source: ArcStr,
) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
let parser = self.as_capturing_parser();
// this will conditionally parse because it's using a CapturingEsParser
parser.parse_program(deno_graph::ast::ParseOptions {
specifier: &module.specifier,
source: module.source.text.clone(),
media_type: module.media_type,
specifier,
source,
media_type,
scope_analysis: false,
})
}
@ -73,8 +87,8 @@ impl ParsedSourceCache {
pub fn remove_or_parse_module(
&self,
specifier: &Url,
source: ArcStr,
media_type: MediaType,
source: ArcStr,
) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
if let Some(parsed_source) = self.remove_parsed_source(specifier) {
if parsed_source.media_type() == media_type

View file

@ -56,6 +56,7 @@ pub type CompilerOptionsTypesRc =
#[cfg(feature = "deno_ast")]
#[derive(Debug)]
pub struct TranspileAndEmitOptions {
pub no_transpile: bool,
pub transpile: deno_ast::TranspileOptions,
pub emit: deno_ast::EmitOptions,
// stored ahead of time so we don't have to recompute this a lot
@ -90,6 +91,11 @@ struct MemoizedValues {
#[derive(Debug, Clone, Default)]
pub struct CompilerOptionsOverrides {
/// Skip transpiling in the loaders.
pub no_transpile: bool,
/// Base to use for the source map. This is useful when bundling
/// and you want to make file urls relative.
pub source_map_base: Option<Url>,
/// Preserve JSX instead of transforming it.
///
/// This may be useful when bundling.
@ -870,6 +876,7 @@ fn compiler_options_to_transpile_and_emit_options(
hasher.finish()
};
Ok(TranspileAndEmitOptions {
no_transpile: overrides.no_transpile,
transpile,
emit,
pre_computed_hash: transpile_and_emit_options_hash,

View file

@ -6,6 +6,7 @@ use std::hash::Hasher;
use anyhow::Error as AnyError;
use deno_ast::EmittedSourceText;
use deno_ast::ModuleKind;
use deno_ast::ParsedSource;
use deno_ast::SourceMapOption;
use deno_ast::SourceRange;
use deno_ast::SourceRanged;
@ -24,11 +25,12 @@ use url::Url;
use crate::cache::EmitCacheRc;
use crate::cache::EmitCacheSys;
use crate::cache::ParsedSourceCache;
use crate::cache::ParsedSourceCacheRc;
use crate::cjs::CjsTrackerRc;
use crate::deno_json::CompilerOptionsResolverRc;
use crate::deno_json::TranspileAndEmitOptions;
use crate::sync::MaybeSend;
use crate::sync::MaybeSync;
#[allow(clippy::disallowed_types)] // ok because we always store source text as Arc<str>
type ArcStr = std::sync::Arc<str>;
@ -79,7 +81,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
if module.media_type.is_emittable() {
futures.push(
self
.emit_parsed_source(
.maybe_emit_source(
&module.specifier,
module.media_type,
ModuleKind::from_is_cjs(
@ -119,50 +121,80 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
Ok(self.emit_cache.get_emit_code(specifier, source_hash))
}
pub async fn emit_parsed_source(
pub async fn maybe_emit_source(
&self,
specifier: &Url,
media_type: MediaType,
module_kind: ModuleKind,
source: &ArcStr,
) -> Result<ArcStr, EmitParsedSourceHelperError> {
if !media_type.is_emittable() {
return Ok(source.clone());
self
.maybe_emit_parsed_source_provider(
ParsedSourceCacheParsedSourceProvider {
parsed_source_cache: self.parsed_source_cache.clone(),
specifier: specifier.clone(),
media_type,
source: source.clone(),
},
module_kind,
)
.await
}
pub async fn maybe_emit_parsed_source(
&self,
parsed_source: deno_ast::ParsedSource,
module_kind: ModuleKind,
) -> Result<ArcStr, EmitParsedSourceHelperError> {
// note: this method is used in deno-js-loader
self
.maybe_emit_parsed_source_provider(parsed_source, module_kind)
.await
}
async fn maybe_emit_parsed_source_provider<
TProvider: ParsedSourceProvider,
>(
&self,
provider: TProvider,
module_kind: ModuleKind,
) -> Result<ArcStr, EmitParsedSourceHelperError> {
// Note: keep this in sync with the sync version below
if !provider.media_type().is_emittable() {
return Ok(provider.into_source());
}
let transpile_and_emit_options = self
.compiler_options_resolver
.for_specifier(specifier)
.for_specifier(provider.specifier())
.transpile_options()?;
if transpile_and_emit_options.no_transpile {
return Ok(provider.into_source());
}
let transpile_options = &transpile_and_emit_options.transpile;
if matches!(media_type, MediaType::Jsx)
if matches!(provider.media_type(), MediaType::Jsx)
&& !transpile_options.transform_jsx
&& !transpile_options.precompile_jsx
{
// jsx disabled, so skip
return Ok(source.clone());
return Ok(provider.into_source());
}
// Note: keep this in sync with the sync version below
let helper = EmitParsedSourceHelper(self);
match helper.pre_emit_parsed_source(
specifier,
provider.specifier(),
module_kind,
transpile_and_emit_options,
source,
provider.source(),
) {
PreEmitResult::Cached(emitted_text) => Ok(emitted_text.into()),
PreEmitResult::NotCached { source_hash } => {
let specifier = provider.specifier().clone();
let emit = {
let parsed_source_cache = self.parsed_source_cache.clone();
let transpile_and_emit_options = transpile_and_emit_options.clone();
let specifier = specifier.clone();
let source = source.clone();
move || {
let parsed_source = provider.parsed_source()?;
transpile(
&parsed_source_cache,
&specifier,
media_type,
parsed_source,
module_kind,
source.clone(),
&transpile_and_emit_options.transpile,
&transpile_and_emit_options.emit,
)
@ -175,7 +207,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
#[cfg(not(feature = "sync"))]
let transpiled_source = emit()?;
helper.post_emit_parsed_source(
specifier,
&specifier,
&transpiled_source,
source_hash,
);
@ -185,18 +217,32 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
}
#[allow(clippy::result_large_err)]
pub fn emit_parsed_source_sync(
pub fn maybe_emit_source_sync(
&self,
specifier: &Url,
media_type: MediaType,
module_kind: deno_ast::ModuleKind,
source: &ArcStr,
) -> Result<String, EmitParsedSourceHelperError> {
) -> Result<ArcStr, EmitParsedSourceHelperError> {
// Note: keep this in sync with the async version above
if !media_type.is_emittable() {
return Ok(source.clone());
}
let transpile_and_emit_options = self
.compiler_options_resolver
.for_specifier(specifier)
.transpile_options()?;
// Note: keep this in sync with the async version above
if transpile_and_emit_options.no_transpile {
return Ok(source.clone());
}
let transpile_options = &transpile_and_emit_options.transpile;
if matches!(media_type, MediaType::Jsx)
&& !transpile_options.transform_jsx
&& !transpile_options.precompile_jsx
{
// jsx disabled, so skip
return Ok(source.clone());
}
let helper = EmitParsedSourceHelper(self);
match helper.pre_emit_parsed_source(
specifier,
@ -204,14 +250,16 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
transpile_and_emit_options,
source,
) {
PreEmitResult::Cached(emitted_text) => Ok(emitted_text),
PreEmitResult::Cached(emitted_text) => Ok(emitted_text.into()),
PreEmitResult::NotCached { source_hash } => {
let transpiled_source = transpile(
&self.parsed_source_cache,
let parsed_source = self.parsed_source_cache.remove_or_parse_module(
specifier,
media_type,
module_kind,
source.clone(),
)?;
let transpiled_source = transpile(
parsed_source,
module_kind,
&transpile_and_emit_options.transpile,
&transpile_and_emit_options.emit,
)?
@ -221,12 +269,12 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
&transpiled_source,
source_hash,
);
Ok(transpiled_source)
Ok(transpiled_source.into())
}
}
}
pub fn emit_parsed_source_for_deno_compile(
pub fn emit_source_for_deno_compile(
&self,
specifier: &Url,
media_type: MediaType,
@ -243,12 +291,14 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
// strip off the path to have more deterministic builds as we don't care
// about the source name because we manually provide the source map to v8
emit_options.source_map_base = Some(deno_path_util::url_parent(specifier));
let source = transpile(
&self.parsed_source_cache,
let parsed_source = self.parsed_source_cache.remove_or_parse_module(
specifier,
media_type,
module_kind,
source.clone(),
)?;
let source = transpile(
parsed_source,
module_kind,
&transpile_and_emit_options.transpile,
&emit_options,
)?;
@ -271,7 +321,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
let source_arc: ArcStr = source_code.into();
let parsed_source = self
.parsed_source_cache
.remove_or_parse_module(specifier, source_arc, media_type)
.remove_or_parse_module(specifier, media_type, source_arc)
.map_err(JsErrorBox::from_err)?;
// HMR doesn't work with embedded source maps for some reason, so set
// the option to not use them (though you should test this out because
@ -340,6 +390,62 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
}
}
trait ParsedSourceProvider: MaybeSend + MaybeSync + Clone + 'static {
fn specifier(&self) -> &Url;
fn media_type(&self) -> MediaType;
fn source(&self) -> &ArcStr;
fn into_source(self) -> ArcStr;
fn parsed_source(self) -> Result<ParsedSource, deno_ast::ParseDiagnostic>;
}
#[derive(Clone)]
struct ParsedSourceCacheParsedSourceProvider {
parsed_source_cache: ParsedSourceCacheRc,
specifier: Url,
media_type: MediaType,
source: ArcStr,
}
impl ParsedSourceProvider for ParsedSourceCacheParsedSourceProvider {
fn specifier(&self) -> &Url {
&self.specifier
}
fn media_type(&self) -> MediaType {
self.media_type
}
fn source(&self) -> &ArcStr {
&self.source
}
fn into_source(self) -> ArcStr {
self.source
}
fn parsed_source(self) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
self.parsed_source_cache.remove_or_parse_module(
&self.specifier,
self.media_type,
self.source.clone(),
)
}
}
impl ParsedSourceProvider for ParsedSource {
fn specifier(&self) -> &Url {
ParsedSource::specifier(self)
}
fn media_type(&self) -> MediaType {
ParsedSource::media_type(self)
}
fn source(&self) -> &ArcStr {
self.text()
}
fn into_source(self) -> ArcStr {
self.text().clone()
}
fn parsed_source(self) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
Ok(self)
}
}
enum PreEmitResult {
Cached(String),
NotCached { source_hash: u64 },
@ -410,18 +516,11 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: EmitterSys>
#[allow(clippy::result_large_err)]
fn transpile(
parsed_source_cache: &ParsedSourceCache,
specifier: &Url,
media_type: MediaType,
parsed_source: ParsedSource,
module_kind: deno_ast::ModuleKind,
source: ArcStr,
transpile_options: &deno_ast::TranspileOptions,
emit_options: &deno_ast::EmitOptions,
) -> Result<EmittedSourceText, EmitParsedSourceHelperError> {
// nothing else needs the parsed source at this point, so remove from
// the cache in order to not transpile owned
let parsed_source = parsed_source_cache
.remove_or_parse_module(specifier, source, media_type)?;
ensure_no_import_assertion(&parsed_source)?;
let transpile_result = parsed_source.transpile(
transpile_options,

View file

@ -197,7 +197,6 @@ pub struct NpmProcessStateOptions {
#[derive(Debug, Default)]
pub struct WorkspaceFactoryOptions {
pub additional_config_file_names: &'static [&'static str],
pub compiler_options_overrides: CompilerOptionsOverrides,
pub config_discovery: ConfigDiscoveryOption,
pub is_package_manager_subcommand: bool,
pub frozen_lockfile: Option<bool>,
@ -661,6 +660,7 @@ impl<TSys: WorkspaceFactorySys> WorkspaceFactory<TSys> {
#[derive(Default)]
pub struct ResolverFactoryOptions {
pub compiler_options_overrides: CompilerOptionsOverrides,
pub is_cjs_resolution_mode: IsCjsResolutionMode,
pub node_analysis_cache: Option<NodeAnalysisCacheRc>,
pub node_code_translator_mode: node_resolver::analyze::NodeCodeTranslatorMode,
@ -847,7 +847,7 @@ impl<TSys: WorkspaceFactorySys> ResolverFactory<TSys> {
self.workspace_factory.workspace_directory_provider()?,
self.node_resolver()?,
&self.workspace_factory.options.config_discovery,
&self.workspace_factory.options.compiler_options_overrides,
&self.options.compiler_options_overrides,
)))
})
}

View file

@ -177,7 +177,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: PreparedModuleLoaderSys>
}) => {
let transpile_result = self
.emitter
.emit_parsed_source(specifier, media_type, ModuleKind::Esm, source)
.maybe_emit_source(specifier, media_type, ModuleKind::Esm, source)
.await?;
// at this point, we no longer need the parsed source in memory, so free it
@ -229,7 +229,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: PreparedModuleLoaderSys>
media_type,
source,
}) => {
let transpile_result = self.emitter.emit_parsed_source_sync(
let transpile_result = self.emitter.maybe_emit_source_sync(
specifier,
media_type,
ModuleKind::Esm,
@ -241,7 +241,7 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: PreparedModuleLoaderSys>
Ok(Some(PreparedModule {
// note: it's faster to provide a string if we know it's a string
source: PreparedModuleSource::ArcStr(transpile_result.into()),
source: PreparedModuleSource::ArcStr(transpile_result),
specifier,
media_type,
}))
@ -412,19 +412,15 @@ impl<TInNpmPackageChecker: InNpmPackageChecker, TSys: PreparedModuleLoaderSys>
media_type: MediaType,
original_source: &ArcStr,
) -> Result<ArcStr, LoadMaybeCjsError> {
let js_source = if media_type.is_emittable() {
self
.emitter
.emit_parsed_source(
specifier,
media_type,
ModuleKind::Cjs,
original_source,
)
.await?
} else {
original_source.clone()
};
let js_source = self
.emitter
.maybe_emit_source(
specifier,
media_type,
ModuleKind::Cjs,
original_source,
)
.await?;
let text = self
.node_code_translator
.translate_cjs_to_esm(specifier, Some(Cow::Borrowed(js_source.as_ref())))