feat(unstable): support bytes and text imports in deno compile (#29924)

Also includes:

- https://github.com/denoland/deno_graph/pull/593

Closes https://github.com/denoland/deno/issues/29903
Closes https://github.com/denoland/deno/issues/29927

.dprint.json

@@ -43,6 +43,7 @@
     "tests/node_compat/test",
     "tests/registry/",
     "tests/specs/bench/default_ts",
+    "tests/specs/compile/bytes_and_text_imports",
     "tests/specs/fmt",
     "tests/specs/lint/bom",
     "tests/specs/lint/default_ts",
@@ -50,7 +51,6 @@
     "tests/specs/publish/no_check_surfaces_syntax_error",
     "tests/specs/run/default_ts",
     "tests/specs/test/default_ts",
-    "tests/testdata/byte_order_mark.ts",
     "tests/testdata/encoding",
     "tests/testdata/file_extensions/ts_with_js_extension.js",
     "tests/testdata/fmt/",
@@ -61,6 +61,7 @@
     "tests/testdata/lint/glob/",
     "tests/testdata/malformed_config/",
     "tests/testdata/run/byte_order_mark.ts",
+    "tests/testdata/run/invalid_utf8.ts",
     "tests/testdata/run/error_syntax_empty_trailing_line.mjs",
     "tests/testdata/run/inline_js_source_map*",
     "tests/testdata/test/markdown_windows.md",

Cargo.lock

@@ -2062,9 +2062,9 @@ dependencies = [
 
 [[package]]
 name = "deno_graph"
-version = "0.96.0"
+version = "0.96.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c8d829b56e6cbd4176981234d1cffe285e0f8a680edbc0cdcfab91db018748c"
+checksum = "0733ed99295ebeddeb0fb33efa41ccd47ccd9481ab1ebe01f2ea8af80204479e"
 dependencies = [
  "async-trait",
  "boxed_error",

Cargo.toml

@@ -64,7 +64,7 @@ deno_core = { version = "0.352.0" }
 deno_cache_dir = "=0.23.0"
 deno_doc = "=0.179.0"
 deno_error = "=0.6.1"
-deno_graph = { version = "=0.96.0", default-features = false }
+deno_graph = { version = "=0.96.2", default-features = false }
 deno_lint = "=0.76.0"
 deno_lockfile = "=0.30.1"
 deno_media_type = { version = "=0.2.9", features = ["module_specifier"] }

cli/factory.rs

@@ -964,6 +964,7 @@ impl CliFactory {
       self.cli_options()?,
       self.deno_dir()?,
       self.emitter()?,
+      self.file_fetcher()?,
       self.http_client_provider(),
       self.npm_resolver().await?,
       self.workspace_resolver().await?.as_ref(),

cli/lib/standalone/binary.rs

@@ -136,9 +136,11 @@ pub enum CjsExportAnalysisEntry {
 const HAS_TRANSPILED_FLAG: u8 = 1 << 0;
 const HAS_SOURCE_MAP_FLAG: u8 = 1 << 1;
 const HAS_CJS_EXPORT_ANALYSIS_FLAG: u8 = 1 << 2;
+const HAS_VALID_UTF8_FLAG: u8 = 1 << 3;
 
 pub struct RemoteModuleEntry<'a> {
   pub media_type: MediaType,
+  pub is_valid_utf8: bool,
   pub data: Cow<'a, [u8]>,
   pub maybe_transpiled: Option<Cow<'a, [u8]>>,
   pub maybe_source_map: Option<Cow<'a, [u8]>>,
@@ -161,6 +163,9 @@ impl<'a> DenoRtSerializable<'a> for RemoteModuleEntry<'a> {
     }
 
     let mut has_data_flags = 0;
+    if self.is_valid_utf8 {
+      has_data_flags |= HAS_VALID_UTF8_FLAG;
+    }
     if self.maybe_transpiled.is_some() {
       has_data_flags |= HAS_TRANSPILED_FLAG;
     }
@@ -203,6 +208,7 @@ impl<'a> DenoRtDeserializable<'a> for RemoteModuleEntry<'a> {
       deserialize_data_if_has_flag(input, has_data_flags, HAS_TRANSPILED_FLAG)?;
     let (input, maybe_source_map) =
       deserialize_data_if_has_flag(input, has_data_flags, HAS_SOURCE_MAP_FLAG)?;
+    let is_valid_utf8 = has_data_flags & HAS_VALID_UTF8_FLAG != 0;
     let (input, maybe_cjs_export_analysis) = deserialize_data_if_has_flag(
       input,
       has_data_flags,
@@ -213,6 +219,7 @@ impl<'a> DenoRtDeserializable<'a> for RemoteModuleEntry<'a> {
       Self {
         media_type,
         data: Cow::Borrowed(data),
+        is_valid_utf8,
         maybe_transpiled,
         maybe_source_map,
         maybe_cjs_export_analysis,

cli/lib/standalone/virtual_fs.rs

@@ -26,6 +26,8 @@ use serde::Deserializer;
 use serde::Serialize;
 use serde::Serializer;
 
+use crate::util::text_encoding::is_valid_utf8;
+
 #[derive(Debug, PartialEq, Eq)]
 pub enum WindowsSystemRootablePath {
   /// The root of the system above any drive letters.
@@ -257,6 +259,8 @@ pub struct VirtualFile {
   pub name: String,
   #[serde(rename = "o")]
   pub offset: OffsetWithLength,
+  #[serde(default, rename = "u", skip_serializing_if = "is_false")]
+  pub is_valid_utf8: bool,
   #[serde(rename = "m", skip_serializing_if = "Option::is_none")]
   pub transpiled_offset: Option<OffsetWithLength>,
   #[serde(rename = "c", skip_serializing_if = "Option::is_none")]
@@ -267,6 +271,10 @@ pub struct VirtualFile {
   pub mtime: Option<u128>, // mtime in milliseconds
 }
 
+fn is_false(value: &bool) -> bool {
+  !value
+}
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct VirtualSymlinkParts(Vec<String>);
 
@@ -766,6 +774,7 @@ impl VfsBuilder {
     log::debug!("Adding file '{}'", path.display());
     let case_sensitivity = self.case_sensitivity;
 
+    let is_valid_utf8 = is_valid_utf8(&options.data);
     let offset_and_len = self.files.add_data(options.data);
     let transpiled_offset = options
       .maybe_transpiled
@@ -790,6 +799,7 @@ impl VfsBuilder {
       || {
         VfsEntry::File(VirtualFile {
           name: name.to_string(),
+          is_valid_utf8,
           offset: offset_and_len,
           transpiled_offset,
           cjs_export_analysis_offset,

cli/lib/util/text_encoding.rs

@@ -3,6 +3,10 @@
 use std::borrow::Cow;
 use std::sync::Arc;
 
+pub fn is_valid_utf8(bytes: &[u8]) -> bool {
+  matches!(String::from_utf8_lossy(bytes), Cow::Borrowed(_))
+}
+
 #[inline(always)]
 pub fn from_utf8_lossy_owned(bytes: Vec<u8>) -> String {
   match String::from_utf8_lossy(&bytes) {

cli/rt/binary.rs

@@ -14,6 +14,7 @@ use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_core::FastString;
+use deno_core::ModuleCodeBytes;
 use deno_core::ModuleSourceCode;
 use deno_core::ModuleType;
 use deno_error::JsError;
@@ -348,12 +349,14 @@ impl StandaloneModules {
       let mut transpiled = None;
       let mut source_map = None;
       let mut cjs_export_analysis = None;
+      let mut is_valid_utf8 = false;
       let bytes = match self.vfs.file_entry(&path) {
         Ok(entry) => {
           let bytes = self
             .vfs
             .read_file_all(entry)
             .map_err(JsErrorBox::from_err)?;
+          is_valid_utf8 = entry.is_valid_utf8;
           transpiled = entry
             .transpiled_offset
             .and_then(|t| self.vfs.read_file_offset_with_len(t).ok());
@@ -379,6 +382,7 @@ impl StandaloneModules {
       Ok(Some(DenoCompileModuleData {
         media_type: MediaType::from_specifier(specifier),
         specifier,
+        is_valid_utf8,
         data: bytes,
         transpiled,
         source_map,
@@ -393,6 +397,7 @@ impl StandaloneModules {
 pub struct DenoCompileModuleData<'a> {
   pub specifier: &'a Url,
   pub media_type: MediaType,
+  pub is_valid_utf8: bool,
   pub data: Cow<'static, [u8]>,
   pub transpiled: Option<Cow<'static, [u8]>>,
   pub source_map: Option<Cow<'static, [u8]>>,
@@ -401,12 +406,18 @@ pub struct DenoCompileModuleData<'a> {
 
 impl<'a> DenoCompileModuleData<'a> {
   pub fn into_parts(self) -> (&'a Url, ModuleType, DenoCompileModuleSource) {
-    fn into_string_unsafe(data: Cow<'static, [u8]>) -> DenoCompileModuleSource {
+    fn into_string_unsafe(
+      is_valid_utf8: bool,
+      data: Cow<'static, [u8]>,
+    ) -> DenoCompileModuleSource {
       match data {
-        Cow::Borrowed(d) => DenoCompileModuleSource::String(
+        Cow::Borrowed(d) if is_valid_utf8 => {
+          DenoCompileModuleSource::String(
             // SAFETY: we know this is a valid utf8 string
             unsafe { std::str::from_utf8_unchecked(d) },
-        ),
+          )
+        }
+        Cow::Borrowed(_) => DenoCompileModuleSource::Bytes(data),
         Cow::Owned(d) => DenoCompileModuleSource::Bytes(Cow::Owned(d)),
       }
     }
@@ -423,8 +434,14 @@ impl<'a> DenoCompileModuleData<'a> {
       | MediaType::Dts
       | MediaType::Dmts
       | MediaType::Dcts
-      | MediaType::Tsx => (ModuleType::JavaScript, into_string_unsafe(data)),
+      | MediaType::Tsx => (
-      MediaType::Json => (ModuleType::Json, into_string_unsafe(data)),
+        ModuleType::JavaScript,
+        into_string_unsafe(self.is_valid_utf8, data),
+      ),
+      MediaType::Json => (
+        ModuleType::Json,
+        into_string_unsafe(self.is_valid_utf8, data),
+      ),
       MediaType::Wasm => {
         (ModuleType::Wasm, DenoCompileModuleSource::Bytes(data))
       }
@@ -441,6 +458,7 @@ impl<'a> DenoCompileModuleData<'a> {
   }
 }
 
+#[derive(Debug)]
 pub enum DenoCompileModuleSource {
   String(&'static str),
   Bytes(Cow<'static, [u8]>),
@@ -448,21 +466,28 @@ pub enum DenoCompileModuleSource {
 
 impl DenoCompileModuleSource {
   pub fn into_for_v8(self) -> ModuleSourceCode {
-    fn into_bytes(data: Cow<'static, [u8]>) -> ModuleSourceCode {
-      ModuleSourceCode::Bytes(match data {
-        Cow::Borrowed(d) => d.into(),
-        Cow::Owned(d) => d.into_boxed_slice().into(),
-      })
-    }
-
     match self {
       // todo(https://github.com/denoland/deno_core/pull/943): store whether
       // the string is ascii or not ahead of time so we can avoid the is_ascii()
       // check in FastString::from_static
       Self::String(s) => ModuleSourceCode::String(FastString::from_static(s)),
-      Self::Bytes(b) => into_bytes(b),
+      Self::Bytes(b) => ModuleSourceCode::Bytes(module_source_into_bytes(b)),
     }
   }
+
+  pub fn into_bytes_for_v8(self) -> ModuleCodeBytes {
+    match self {
+      DenoCompileModuleSource::String(text) => text.as_bytes().into(),
+      DenoCompileModuleSource::Bytes(b) => module_source_into_bytes(b),
+    }
+  }
+}
+
+fn module_source_into_bytes(data: Cow<'static, [u8]>) -> ModuleCodeBytes {
+  match data {
+    Cow::Borrowed(d) => d.into(),
+    Cow::Owned(d) => d.into_boxed_slice().into(),
+  }
 }
 
 #[derive(Debug, Error, JsError)]
@@ -558,6 +583,7 @@ impl RemoteModulesStore {
               self.specifiers.get_specifier(specifier).unwrap()
             },
             media_type: entry.media_type,
+            is_valid_utf8: entry.is_valid_utf8,
             data: handle_cow_ref(&entry.data),
             transpiled: entry.maybe_transpiled.as_ref().map(handle_cow_ref),
             source_map: entry.maybe_source_map.as_ref().map(handle_cow_ref),

cli/rt/run.rs

@@ -17,6 +17,7 @@ use deno_core::v8_set_flags;
 use deno_core::FastString;
 use deno_core::ModuleLoader;
 use deno_core::ModuleSourceCode;
+use deno_core::ModuleType;
 use deno_core::RequestedModuleType;
 use deno_core::ResolutionKind;
 use deno_core::SourceCodeCacheInfo;
@@ -370,7 +371,7 @@ impl ModuleLoader for EmbeddedModuleLoader {
     original_specifier: &Url,
     maybe_referrer: Option<&Url>,
     _is_dynamic: bool,
-    _requested_module_type: RequestedModuleType,
+    requested_module_type: RequestedModuleType,
   ) -> deno_core::ModuleLoadResponse {
     if original_specifier.scheme() == "data" {
       let data_url_text =
@@ -423,6 +424,36 @@ impl ModuleLoader for EmbeddedModuleLoader {
 
     match self.shared.modules.read(original_specifier) {
       Ok(Some(module)) => {
+        match requested_module_type {
+          RequestedModuleType::Text | RequestedModuleType::Bytes => {
+            let module_source = DenoCompileModuleSource::Bytes(module.data);
+            return deno_core::ModuleLoadResponse::Sync(Ok(
+              deno_core::ModuleSource::new_with_redirect(
+                match requested_module_type {
+                  RequestedModuleType::Text => ModuleType::Text,
+                  RequestedModuleType::Bytes => ModuleType::Bytes,
+                  _ => unreachable!(),
+                },
+                match requested_module_type {
+                  RequestedModuleType::Text => module_source.into_for_v8(),
+                  RequestedModuleType::Bytes => {
+                    ModuleSourceCode::Bytes(module_source.into_bytes_for_v8())
+                  }
+                  _ => unreachable!(),
+                },
+                original_specifier,
+                module.specifier,
+                None,
+              ),
+            ));
+          }
+          RequestedModuleType::Other(_)
+          | RequestedModuleType::None
+          | RequestedModuleType::Json => {
+            // ignore
+          }
+        }
+
         let media_type = module.media_type;
         let (module_specifier, module_type, module_source) =
           module.into_parts();

cli/standalone/binary.rs

@@ -44,6 +44,7 @@ use deno_lib::standalone::virtual_fs::VirtualDirectoryEntries;
 use deno_lib::standalone::virtual_fs::WindowsSystemRootablePath;
 use deno_lib::standalone::virtual_fs::DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME;
 use deno_lib::util::hash::FastInsecureHasher;
+use deno_lib::util::text_encoding::is_valid_utf8;
 use deno_lib::util::v8::construct_v8_flags;
 use deno_lib::version::DENO_VERSION_INFO;
 use deno_npm::resolution::SerializedNpmResolutionSnapshot;
@@ -51,6 +52,9 @@ use deno_npm::NpmSystemInfo;
 use deno_path_util::fs::atomic_write_file_with_retries;
 use deno_path_util::url_from_directory_path;
 use deno_path_util::url_to_file_path;
+use deno_resolver::file_fetcher::FetchLocalOptions;
+use deno_resolver::file_fetcher::FetchOptions;
+use deno_resolver::file_fetcher::FetchPermissionsOptionRef;
 use deno_resolver::workspace::WorkspaceResolver;
 use indexmap::IndexMap;
 use node_resolver::analyze::ResolvedCjsAnalysis;
@@ -61,6 +65,7 @@ use crate::args::CliOptions;
 use crate::args::CompileFlags;
 use crate::cache::DenoDir;
 use crate::emit::Emitter;
+use crate::file_fetcher::CliFileFetcher;
 use crate::http_util::HttpClientProvider;
 use crate::node::CliCjsModuleExportAnalyzer;
 use crate::npm::CliNpmResolver;
@@ -197,6 +202,7 @@ pub struct DenoCompileBinaryWriter<'a> {
   cli_options: &'a CliOptions,
   deno_dir: &'a DenoDir,
   emitter: &'a Emitter,
+  file_fetcher: &'a CliFileFetcher,
   http_client_provider: &'a HttpClientProvider,
   npm_resolver: &'a CliNpmResolver,
   workspace_resolver: &'a WorkspaceResolver<CliSys>,
@@ -211,6 +217,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     cli_options: &'a CliOptions,
     deno_dir: &'a DenoDir,
     emitter: &'a Emitter,
+    file_fetcher: &'a CliFileFetcher,
     http_client_provider: &'a HttpClientProvider,
     npm_resolver: &'a CliNpmResolver,
     workspace_resolver: &'a WorkspaceResolver<CliSys>,
@@ -222,6 +229,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       cli_options,
       deno_dir,
       emitter,
+      file_fetcher,
       http_client_provider,
       npm_resolver,
       workspace_resolver,
@@ -409,6 +417,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     let mut specifier_store = SpecifierStore::with_capacity(specifiers_count);
     let mut remote_modules_store =
       SpecifierDataStore::with_capacity(specifiers_count);
+    let mut asset_module_urls = graph.asset_module_urls();
     // todo(dsherret): transpile and analyze CJS in parallel
     for module in graph.modules() {
       if module.specifier().scheme() == "data" {
@@ -420,7 +429,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       let (maybe_original_source, media_type) = match module {
         deno_graph::Module::Js(m) => {
           let specifier = &m.specifier;
-          let original_bytes = m.source.text.as_bytes();
+          let original_bytes = match m.source.try_get_original_bytes() {
+            Some(bytes) => bytes,
+            None => self.load_asset_bypass_permissions(specifier).await?.source,
+          };
           if self.cjs_tracker.is_maybe_cjs(specifier, m.media_type)? {
             if self.cjs_tracker.is_cjs_with_known_is_script(
               specifier,
@@ -466,16 +478,26 @@ impl<'a> DenoCompileBinaryWriter<'a> {
           (Some(original_bytes), m.media_type)
         }
         deno_graph::Module::Json(m) => {
-          (Some(m.source.text.as_bytes()), m.media_type)
+          let original_bytes = match m.source.try_get_original_bytes() {
+            Some(bytes) => bytes,
+            None => {
+              self
+                .load_asset_bypass_permissions(&m.specifier)
+                .await?
+                .source
+            }
+          };
+          (Some(original_bytes), m.media_type)
         }
         deno_graph::Module::Wasm(m) => {
-          (Some(m.source.as_ref()), MediaType::Wasm)
+          (Some(m.source.clone()), MediaType::Wasm)
         }
         deno_graph::Module::Npm(_)
         | deno_graph::Module::Node(_)
         | deno_graph::Module::External(_) => (None, MediaType::Unknown),
       };
       if let Some(original_source) = maybe_original_source {
+        asset_module_urls.swap_remove(module.specifier());
         let maybe_cjs_export_analysis = maybe_cjs_analysis
           .as_ref()
           .map(bincode::serialize)
@@ -505,7 +527,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
             specifier_id,
             RemoteModuleEntry {
               media_type,
-              data: Cow::Borrowed(original_source),
+              is_valid_utf8: is_valid_utf8(&original_source),
+              data: Cow::Owned(original_source.to_vec()),
               maybe_transpiled: maybe_transpiled.map(Cow::Owned),
               maybe_source_map: maybe_source_map.map(Cow::Owned),
               maybe_cjs_export_analysis: maybe_cjs_export_analysis
@@ -516,6 +539,42 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       }
     }
 
+    for url in asset_module_urls {
+      if graph.try_get(url).is_err() {
+        // skip because there was an error loading this module
+        continue;
+      }
+      match url.scheme() {
+        "file" => {
+          let file_path = deno_path_util::url_to_file_path(url)?;
+          vfs.add_file_at_path(&file_path)?;
+        }
+        "http" | "https" => {
+          let specifier_id = specifier_store.get_or_add(url);
+          if !remote_modules_store.contains(specifier_id) {
+            // it's ok to bypass permissions here because we verified the module
+            // loaded successfully in the graph
+            let file = self.load_asset_bypass_permissions(url).await?;
+            remote_modules_store.add(
+              specifier_id,
+              RemoteModuleEntry {
+                media_type: MediaType::from_specifier_and_headers(
+                  &file.url,
+                  file.maybe_headers.as_ref(),
+                ),
+                is_valid_utf8: is_valid_utf8(&file.source),
+                data: Cow::Owned(file.source.to_vec()),
+                maybe_cjs_export_analysis: None,
+                maybe_source_map: None,
+                maybe_transpiled: None,
+              },
+            );
+          }
+        }
+        _ => {}
+      }
+    }
+
     let mut redirects_store =
       SpecifierDataStore::with_capacity(graph.redirects.len());
     for (from, to) in &graph.redirects {
@@ -756,6 +815,32 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       .context("Writing binary bytes")
   }
 
+  async fn load_asset_bypass_permissions(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<
+    deno_cache_dir::file_fetcher::File,
+    deno_resolver::file_fetcher::FetchError,
+  > {
+    self
+      .file_fetcher
+      .fetch_with_options(
+        specifier,
+        FetchPermissionsOptionRef::AllowAll,
+        FetchOptions {
+          local: FetchLocalOptions {
+            include_mtime: false,
+          },
+          maybe_auth: None,
+          maybe_accept: None,
+          maybe_cache_setting: Some(
+            &deno_cache_dir::file_fetcher::CacheSetting::Use,
+          ),
+        },
+      )
+      .await
+  }
+
   fn fill_npm_vfs(&self, builder: &mut VfsBuilder) -> Result<(), AnyError> {
     fn maybe_warn_different_system(system_info: &NpmSystemInfo) {
       if system_info != &NpmSystemInfo::default() {

tests/specs/compile/bytes_and_text_imports/allow_import/__test__.jsonc

@@ -0,0 +1,8 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "args": "compile --unstable-raw-imports --output bin main.ts",
+    "output": "compile.out",
+    "exitCode": 1
+  }]
+}

tests/specs/compile/bytes_and_text_imports/allow_import/compile.out

@@ -0,0 +1,2 @@
+error: Requires import access to "localhost:4545", run again with the --allow-import flag
+    at file:///[WILDLINE]

tests/specs/compile/bytes_and_text_imports/allow_import/main.ts

@@ -0,0 +1,3 @@
+import bytes from "http://localhost:4545/run/invalid_utf8.ts" with { type: "text" };
+
+console.log(bytes);

tests/specs/compile/bytes_and_text_imports/allow_import_dynamic_branch/__test__.jsonc

@@ -0,0 +1,12 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "args": "compile --unstable-raw-imports --output bin main.ts",
+    "output": "compile.out"
+  }, {
+    "commandName": "./bin",
+    "args": [],
+    "output": "run.out",
+    "exitCode": 1
+  }]
+}

tests/specs/compile/bytes_and_text_imports/allow_import_dynamic_branch/branch.ts

@@ -0,0 +1,3 @@
+import bytes from "http://localhost:4545/run/invalid_utf8.ts" with { type: "text" };
+
+console.log(bytes);

tests/specs/compile/bytes_and_text_imports/allow_import_dynamic_branch/compile.out

@@ -0,0 +1,11 @@
+[# Warning! It should not show that it downloaded anything here]
+Check file:///[WILDLINE]
+Compile file:///[WILDLINE]
+
+Embedded Files
+
+[WILDLINE]
+├── branch.ts ([WILDLINE])
+└── main.ts ([WILDLINE])
+
+[WILDCARD]

tests/specs/compile/bytes_and_text_imports/allow_import_dynamic_branch/main.ts

@@ -0,0 +1 @@
+await import("./branch.ts");

tests/specs/compile/bytes_and_text_imports/allow_import_dynamic_branch/run.out

@@ -0,0 +1,5 @@
+[# This error is not ideal, but ok for now. It should say the permission issue instead]
+error: Uncaught (in promise) TypeError: Module not found: http://localhost:4545/run/invalid_utf8.ts
+await import("./branch.ts");
+^
+    at async file:///[WILDLINE]

tests/specs/compile/bytes_and_text_imports/basic/__test__.jsonc

@@ -0,0 +1,11 @@
+{
+  "tempDir": true,
+  "steps": [{
+    "args": "compile --unstable-raw-imports --allow-import --output bin main.ts",
+    "output": "compile.out"
+  }, {
+    "commandName": "./bin",
+    "args": [],
+    "output": "run.out"
+  }]
+}

tests/specs/compile/bytes_and_text_imports/basic/add_bom.ts

@@ -0,0 +1,4 @@
+// this file has a BOM
+export function add(a: number, b: number) {
+  return a + b;
+}

tests/specs/compile/bytes_and_text_imports/basic/compile.out

@@ -0,0 +1,13 @@
+Download http://localhost:4545/run/invalid_utf8.ts
+Check file:///[WILDLINE]/main.ts
+Compile file:///[WILDLINE]/main.ts to [WILDLINE]
+
+Embedded Files
+
+[WILDLINE]
+├── add_bom.ts ([WILDLINE])
+├── hello_bom.txt ([WILDLINE])
+├── lossy.ts ([WILDLINE])
+└── main.ts ([WILDLINE])
+
+[WILDCARD]

tests/specs/compile/bytes_and_text_imports/basic/hello_bom.txt

@@ -0,0 +1 @@
+hello

tests/specs/compile/bytes_and_text_imports/basic/lossy.ts

@@ -0,0 +1 @@
+console.log("This is a test: À");

tests/specs/compile/bytes_and_text_imports/basic/main.ts

@@ -0,0 +1,21 @@
+import hello from "./hello_bom.txt" with { type: "text" };
+import helloBytes from "./hello_bom.txt" with { type: "bytes" };
+import { add } from "./add_bom.ts";
+import addBytes from "./add_bom.ts" with { type: "bytes" };
+import addText from "./add_bom.ts" with { type: "text" };
+import "./lossy.ts";
+import invalidUtf8Bytes from "./lossy.ts" with { type: "bytes" };
+import invalidUtf8Text from "./lossy.ts" with { type: "text" };
+import "http://localhost:4545/run/invalid_utf8.ts";
+import remoteInvalidUtf8Bytes from "http://localhost:4545/run/invalid_utf8.ts" with { type: "bytes" };
+import removeInvalidUtf8Text from "http://localhost:4545/run/invalid_utf8.ts" with { type: "text" };
+
+console.log(hello, hello.length);
+console.log(helloBytes, helloBytes.length);
+console.log(addText, addText.length);
+console.log(addBytes, addBytes.length);
+console.log(invalidUtf8Bytes, invalidUtf8Bytes.length);
+console.log(invalidUtf8Text, invalidUtf8Text.length);
+console.log(remoteInvalidUtf8Bytes, remoteInvalidUtf8Bytes.length);
+console.log(removeInvalidUtf8Text, removeInvalidUtf8Text.length);
+console.log(add(1, 2));

tests/specs/compile/bytes_and_text_imports/basic/run.out

@@ -0,0 +1,42 @@
+This is a test: <20>
+This is a test: <20>
+hello 5
+Uint8Array(8) [
+  239, 187, 191,
+  104, 101, 108,
+  108, 111
+] 8
+// this file has a BOM
+export function add(a: number, b: number) {
+  return a + b;
+}
+ 85
+Uint8Array(88) [
+  239, 187, 191,  47,  47,  32, 116, 104, 105, 115,  32, 102,
+  105, 108, 101,  32, 104,  97, 115,  32,  97,  32,  66,  79,
+   77,  10, 101, 120, 112, 111, 114, 116,  32, 102, 117, 110,
+   99, 116, 105, 111, 110,  32,  97, 100, 100,  40,  97,  58,
+   32, 110, 117, 109,  98, 101, 114,  44,  32,  98,  58,  32,
+  110, 117, 109,  98, 101, 114,  41,  32, 123,  10,  32,  32,
+  114, 101, 116, 117, 114, 110,  32,  97,  32,  43,  32,  98,
+   59,  10, 125,  10
+] 88
+Uint8Array(34) [
+   99, 111, 110, 115, 111, 108, 101,  46,
+  108, 111, 103,  40,  34,  84, 104, 105,
+  115,  32, 105, 115,  32,  97,  32, 116,
+  101, 115, 116,  58,  32, 192,  34,  41,
+   59,  10
+] 34
+console.log("This is a test: <20>");
+ 34
+Uint8Array(34) [
+   99, 111, 110, 115, 111, 108, 101,  46,
+  108, 111, 103,  40,  34,  84, 104, 105,
+  115,  32, 105, 115,  32,  97,  32, 116,
+  101, 115, 116,  58,  32, 192,  34,  41,
+   59,  10
+] 34
+console.log("This is a test: <20>");
+ 34
+3

tests/testdata/run/invalid_utf8.ts

@@ -0,0 +1 @@
+console.log("This is a test: À");