Refactor snapshot build (#2825)

Instead of using core/snapshot_creator.rs, instead two crates are
introduced which allow building the snapshot during build.rs.

Rollup is removed and replaced with our own bundler. This removes
the Node build dependency. Modules in //js now use Deno-style imports
with file extensions, rather than Node style extensionless imports.

This improves incremental build time when changes are made to //js files
by about 40 seconds.

deno_typescript/BUILD.gn

@@ -0,0 +1,22 @@
+import("//build_extra/rust/rust.gni")
+
+rust_rlib("deno_typescript") {
+  source_root = "lib.rs"
+  generated_source_dir = "."
+  extern = [
+    {
+      label = "../core:deno"
+      crate_name = "deno"
+      crate_type = "rlib"
+    },
+    {
+      label = "$rust_build:serde_derive"
+      crate_name = "serde_derive"
+      crate_type = "proc_macro"
+    },
+  ]
+  extern_rlib = [
+    "serde_json",
+    "serde",
+  ]
+}

deno_typescript/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "deno_typescript"
+version = "0.0.3"
+license = "MIT"
+description = "To compile TypeScript to a snapshot during build.rs"
+repository = "https://github.com/ry/deno_typescript"
+authors = ["Ryan Dahl <ry@tinyclouds.org>"]
+edition = "2018"
+
+[lib]
+path = "lib.rs"
+
+[dependencies]
+deno = { path = "../core" }
+serde_json = "1.0"
+serde = { version = "1.0", features = ["derive"] }

deno_typescript/README.md

@@ -0,0 +1,8 @@
+This crate provides utilies to compile typescript, bundle it up, and create a V8
+snapshot, all during build. This allows users to startup fast.
+
+The cli_snapshots crate, neighboring this one uses deno_typescript at build
+time.
+
+This crate does not depend on Node, Python, nor any other external dependencies
+besides those listed as such in Cargo.toml.

deno_typescript/amd_runtime.js

@@ -0,0 +1,39 @@
+// A very very basic AMD preamble to support the output of TypeScript outFile
+// bundles.
+let require, define;
+
+(function() {
+  const modules = new Map();
+
+  function println(first, ...s) {
+    Deno.core.print(first + " " + s.map(JSON.stringify).join(" ") + "\n");
+  }
+
+  function createOrLoadModule(name) {
+    if (!modules.has(name)) {
+      const m = { name, exports: {} };
+      modules.set(name, m);
+    }
+    return modules.get(name);
+  }
+
+  require = name => {
+    return createOrLoadModule(name).exports;
+  };
+
+  define = (name, deps, factory) => {
+    const currentModule = createOrLoadModule(name);
+    const localExports = currentModule.exports;
+    const args = deps.map(dep => {
+      if (dep === "require") {
+        return require;
+      } else if (dep === "exports") {
+        return localExports;
+      } else {
+        const depModule = createOrLoadModule(dep);
+        return depModule.exports;
+      }
+    });
+    factory(...args);
+  };
+})();

deno_typescript/compiler_main.js

@@ -0,0 +1,320 @@
+// Because we're bootstrapping the TS compiler without dependencies on Node,
+// this is written in JS.
+
+const ASSETS = "$asset$";
+
+let replacements;
+
+function main(configText, rootNames, replacements_) {
+  println(`>>> ts version ${ts.version}`);
+  println(`>>> rootNames ${rootNames}`);
+
+  replacements = replacements_;
+  replacements["DENO_REPLACE_TS_VERSION"] = ts.version;
+  println(`>>> replacements ${JSON.stringify(replacements)}`);
+
+  const host = new Host();
+
+  assert(rootNames.length > 0);
+
+  let { options, diagnostics } = configure(configText);
+  handleDiagnostics(host, diagnostics);
+
+  println(`>>> TS config: ${JSON.stringify(options)}`);
+
+  const program = ts.createProgram(rootNames, options, host);
+
+  diagnostics = ts.getPreEmitDiagnostics(program).filter(({ code }) => {
+    // TS2691: An import path cannot end with a '.ts' extension. Consider
+    // importing 'bad-module' instead.
+    if (code === 2691) return false;
+    // TS5009: Cannot find the common subdirectory path for the input files.
+    if (code === 5009) return false;
+    return true;
+  });
+  handleDiagnostics(host, diagnostics);
+
+  const emitResult = program.emit();
+  handleDiagnostics(host, emitResult.diagnostics);
+
+  dispatch("setEmitResult", emitResult);
+}
+
+function println(...s) {
+  Deno.core.print(s.join(" ") + "\n");
+}
+
+function unreachable() {
+  throw Error("unreachable");
+}
+
+function assert(cond) {
+  if (!cond) {
+    throw Error("assert");
+  }
+}
+
+// decode(Uint8Array): string
+function decodeAscii(ui8) {
+  let out = "";
+  for (let i = 0; i < ui8.length; i++) {
+    out += String.fromCharCode(ui8[i]);
+  }
+  return out;
+}
+
+function encode(str) {
+  const charCodes = str.split("").map(c => c.charCodeAt(0));
+  const ui8 = new Uint8Array(charCodes);
+  return ui8;
+}
+
+// Warning! The op_id values below are shared between this code and
+// the Rust side. Update with care!
+const ops = {
+  readFile: 49,
+  exit: 50,
+  writeFile: 51,
+  resolveModuleNames: 52,
+  setEmitResult: 53
+};
+
+// interface CompilerHost extends ModuleResolutionHost {
+class Host {
+  // fileExists(fileName: string): boolean;
+  fileExists(fileName) {
+    return true;
+  }
+
+  // readFile(fileName: string): string | undefined;
+  readFile() {
+    unreachable();
+  }
+
+  // trace?(s: string): void;
+  // directoryExists?(directoryName: string): boolean;
+  // realpath?(path: string): string;
+  // getCurrentDirectory?(): string;
+  // getDirectories?(path: string): string[];
+
+  // useCaseSensitiveFileNames(): boolean;
+  useCaseSensitiveFileNames() {
+    return false;
+  }
+
+  // getDefaultLibFileName(options: CompilerOptions): string;
+  getDefaultLibFileName(options) {
+    return "lib.deno_core.d.ts";
+  }
+
+  // getDefaultLibLocation?(): string;
+  getDefaultLibLocation() {
+    return ASSETS;
+  }
+
+  // getCurrentDirectory(): string;
+  getCurrentDirectory() {
+    return ".";
+  }
+
+  // getCanonicalFileName(fileName: string): string
+  getCanonicalFileName(fileName) {
+    unreachable();
+  }
+
+  // getSourceFile(fileName: string, languageVersion: ScriptTarget, onError?:
+  // (message: string) => void, shouldCreateNewSourceFile?: boolean): SourceFile
+  // | undefined;
+  getSourceFile(fileName, languageVersion, onError, shouldCreateNewSourceFile) {
+    assert(!shouldCreateNewSourceFile); // We haven't yet encountered this.
+
+    // This hacks around the fact that TypeScript tries to magically guess the
+    // d.ts filename.
+    if (fileName.startsWith("$typeRoots$")) {
+      assert(fileName.startsWith("$typeRoots$/"));
+      assert(fileName.endsWith("/index.d.ts"));
+      fileName = fileName
+        .replace("$typeRoots$/", "")
+        .replace("/index.d.ts", "");
+    }
+
+    let { sourceCode, moduleName } = dispatch("readFile", {
+      fileName,
+      languageVersion,
+      shouldCreateNewSourceFile
+    });
+
+    // TODO(ry) A terrible hack. Please remove ASAP.
+    if (fileName.endsWith("typescript.d.ts")) {
+      sourceCode = sourceCode.replace("export = ts;", "");
+    }
+
+    // TODO(ry) A terrible hack. Please remove ASAP.
+    for (let key of Object.keys(replacements)) {
+      let val = replacements[key];
+      sourceCode = sourceCode.replace(key, val);
+    }
+
+    let sourceFile = ts.createSourceFile(fileName, sourceCode, languageVersion);
+    sourceFile.moduleName = moduleName;
+    return sourceFile;
+  }
+
+  /*
+    writeFile(
+      fileName: string,
+      data: string,
+      writeByteOrderMark: boolean,
+      onError?: (message: string) => void,
+      sourceFiles?: ReadonlyArray<ts.SourceFile>
+    ): void
+  */
+  writeFile(
+    fileName,
+    data,
+    writeByteOrderMark,
+    onError = null,
+    sourceFiles = null
+  ) {
+    const moduleName = sourceFiles[sourceFiles.length - 1].moduleName;
+    return dispatch("writeFile", { fileName, moduleName, data });
+  }
+
+  // getSourceFileByPath?(fileName: string, path: Path, languageVersion: ScriptTarget, onError?: (message: string) => void, shouldCreateNewSourceFile?: boolean): SourceFile | undefined;
+  getSourceFileByPath(
+    fileName,
+    path,
+    languageVersion,
+    onError,
+    shouldCreateNewSourceFile
+  ) {
+    unreachable();
+  }
+
+  // getCancellationToken?(): CancellationToken;
+  getCancellationToken() {
+    unreachable();
+  }
+
+  // getCanonicalFileName(fileName: string): string;
+  getCanonicalFileName(fileName) {
+    return fileName;
+  }
+
+  // getNewLine(): string
+  getNewLine() {
+    return "\n";
+  }
+
+  // readDirectory?(rootDir: string, extensions: ReadonlyArray<string>, excludes: ReadonlyArray<string> | undefined, includes: ReadonlyArray<string>, depth?: number): string[];
+  readDirectory() {
+    unreachable();
+  }
+
+  // resolveModuleNames?(
+  //   moduleNames: string[],
+  //   containingFile: string,
+  //   reusedNames?: string[],
+  //   redirectedReference?: ResolvedProjectReference
+  // ): (ResolvedModule | undefined)[];
+  resolveModuleNames(moduleNames, containingFile) {
+    const resolvedNames = dispatch("resolveModuleNames", {
+      moduleNames,
+      containingFile
+    });
+    const r = resolvedNames.map(resolvedFileName => {
+      const extension = getExtension(resolvedFileName);
+      return { resolvedFileName, extension };
+    });
+    return r;
+  }
+
+  // resolveTypeReferenceDirectives?(typeReferenceDirectiveNames: string[], containingFile: string, redirectedReference?: ResolvedProjectReference): (ResolvedTypeReferenceDirective | undefined)[];
+  /*
+  resolveTypeReferenceDirectives() {
+    unreachable();
+  }
+  */
+
+  // getEnvironmentVariable?(name: string): string | undefined;
+  getEnvironmentVariable() {
+    unreachable();
+  }
+
+  // createHash?(data: string): string;
+  createHash() {
+    unreachable();
+  }
+
+  // getParsedCommandLine?(fileName: string): ParsedCommandLine | undefined;
+  getParsedCommandLine() {
+    unreachable();
+  }
+}
+
+function configure(configurationText) {
+  const { config, error } = ts.parseConfigFileTextToJson(
+    "tsconfig.json",
+    configurationText
+  );
+  if (error) {
+    return { diagnostics: [error] };
+  }
+  const { options, errors } = ts.convertCompilerOptionsFromJson(
+    config.compilerOptions,
+    ""
+  );
+  return {
+    options,
+    diagnostics: errors.length ? errors : undefined
+  };
+}
+
+function dispatch(opName, obj) {
+  const s = JSON.stringify(obj);
+  const msg = encode(s);
+  const resUi8 = Deno.core.dispatch(ops[opName], msg);
+  const resStr = decodeAscii(resUi8);
+  const res = JSON.parse(resStr);
+  if (!res["ok"]) {
+    throw Error(`${opName} failed ${res["err"]}. Args: ${JSON.stringify(obj)}`);
+  }
+  return res["ok"];
+}
+
+function exit(code) {
+  dispatch("exit", { code });
+  unreachable();
+}
+
+// Maximum number of diagnostics to display.
+const MAX_ERRORS = 5;
+
+function handleDiagnostics(host, diagnostics) {
+  if (diagnostics && diagnostics.length) {
+    let rest = 0;
+    if (diagnostics.length > MAX_ERRORS) {
+      rest = diagnostics.length - MAX_ERRORS;
+      diagnostics = diagnostics.slice(0, MAX_ERRORS);
+    }
+    const msg = ts.formatDiagnosticsWithColorAndContext(diagnostics, host);
+    println(msg);
+    if (rest) {
+      println(`And ${rest} other errors.`);
+    }
+    exit(1);
+  }
+}
+
+/** Returns the TypeScript Extension enum for a given media type. */
+function getExtension(fileName) {
+  if (fileName.endsWith(".d.ts")) {
+    return ts.Extension.Dts;
+  } else if (fileName.endsWith(".ts")) {
+    return ts.Extension.Ts;
+  } else if (fileName.endsWith(".js")) {
+    return ts.Extension.Js;
+  } else {
+    throw TypeError(`Cannot resolve extension for ${fileName}`);
+  }
+}

deno_typescript/lib.deno_core.d.ts

@@ -0,0 +1,66 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+
+// This file contains APIs that are introduced into the global namespace by
+// Deno core.  These are not intended to be used directly by runtime users of
+// Deno and therefore do not flow through to the runtime type library.
+
+declare interface MessageCallback {
+  (opId: number, msg: Uint8Array): void;
+}
+
+interface EvalErrorInfo {
+  // Is the object thrown a native Error?
+  isNativeError: boolean;
+  // Was the error happened during compilation?
+  isCompileError: boolean;
+  // The actual thrown entity
+  // (might be an Error or anything else thrown by the user)
+  // If isNativeError is true, this is an Error
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  thrown: any;
+}
+
+declare interface DenoCore {
+  print(s: string, is_err?: boolean);
+  dispatch(
+    opId: number,
+    control: Uint8Array,
+    zeroCopy?: ArrayBufferView | null
+  ): Uint8Array | null;
+  setAsyncHandler(cb: MessageCallback): void;
+  sharedQueue: {
+    head(): number;
+    numRecords(): number;
+    size(): number;
+    push(buf: Uint8Array): boolean;
+    reset(): void;
+    shift(): Uint8Array | null;
+  };
+
+  recv(cb: MessageCallback): void;
+
+  send(
+    opId: number,
+    control: null | ArrayBufferView,
+    data?: ArrayBufferView
+  ): null | Uint8Array;
+
+  print(x: string, isErr?: boolean): void;
+
+  shared: SharedArrayBuffer;
+
+  /** Evaluate provided code in the current context.
+   * It differs from eval(...) in that it does not create a new context.
+   * Returns an array: [output, errInfo].
+   * If an error occurs, `output` becomes null and `errInfo` is non-null.
+   */
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  evalContext(code: string): [any, EvalErrorInfo | null];
+
+  errorToJSON: (e: Error) => string;
+}
+
+declare interface DenoInterface {
+  core: DenoCore;
+}
+declare var Deno: DenoInterface;

deno_typescript/lib.rs

@@ -0,0 +1,288 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+extern crate deno;
+extern crate serde;
+extern crate serde_json;
+
+mod ops;
+use deno::js_check;
+pub use deno::v8_set_flags;
+use deno::ErrBox;
+use deno::Isolate;
+use deno::ModuleSpecifier;
+use deno::StartupData;
+pub use ops::EmitResult;
+use ops::WrittenFile;
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+static TYPESCRIPT_CODE: &str =
+  include_str!("../third_party/node_modules/typescript/lib/typescript.js");
+static COMPILER_CODE: &str = include_str!("compiler_main.js");
+static AMD_RUNTIME_CODE: &str = include_str!("amd_runtime.js");
+
+#[derive(Debug)]
+pub struct TSState {
+  bundle: bool,
+  exit_code: i32,
+  emit_result: Option<EmitResult>,
+  /// A list of files emitted by typescript. WrittenFile is tuple of the form
+  /// (url, corresponding_module, source_code)
+  written_files: Vec<WrittenFile>,
+}
+
+impl TSState {
+  fn main_module_name(&self) -> String {
+    // Assuming that TypeScript has emitted the main file last.
+    self.written_files.last().unwrap().module_name.clone()
+  }
+}
+
+pub struct TSIsolate {
+  isolate: Isolate,
+  state: Arc<Mutex<TSState>>,
+}
+
+impl TSIsolate {
+  fn new(bundle: bool) -> TSIsolate {
+    let mut isolate = Isolate::new(StartupData::None, false);
+    js_check(isolate.execute("assets/typescript.js", TYPESCRIPT_CODE));
+    js_check(isolate.execute("compiler_main.js", COMPILER_CODE));
+
+    let state = Arc::new(Mutex::new(TSState {
+      bundle,
+      exit_code: 0,
+      emit_result: None,
+      written_files: Vec::new(),
+    }));
+    let state_ = state.clone();
+    isolate.set_dispatch(move |op_id, control_buf, zero_copy_buf| {
+      assert!(zero_copy_buf.is_none()); // zero_copy_buf unused in compiler.
+      let mut s = state_.lock().unwrap();
+      ops::dispatch_op(&mut s, op_id, control_buf)
+    });
+    TSIsolate { isolate, state }
+  }
+
+  // TODO(ry) Instead of Result<Arc<Mutex<TSState>>, ErrBox>, return something
+  // like Result<TSState, ErrBox>. I think it would be nicer if this function
+  // consumes TSIsolate.
+  /// Compiles each module to ESM. Doesn't write any files to disk.
+  /// Passes all output via state.
+  fn compile(
+    mut self,
+    config_json: &serde_json::Value,
+    root_names: Vec<String>,
+  ) -> Result<Arc<Mutex<TSState>>, ErrBox> {
+    let root_names_json = serde_json::json!(root_names).to_string();
+    let source = &format!(
+      "main({:?}, {}, {})",
+      config_json.to_string(),
+      root_names_json,
+      preprocessor_replacements_json()
+    );
+    self.isolate.execute("<anon>", source)?;
+    Ok(self.state.clone())
+  }
+}
+
+pub fn compile_bundle(
+  bundle: &Path,
+  root_names: Vec<PathBuf>,
+) -> Result<Arc<Mutex<TSState>>, ErrBox> {
+  let ts_isolate = TSIsolate::new(true);
+
+  let config_json = serde_json::json!({
+    "compilerOptions": {
+      "declaration": true,
+      "lib": ["esnext"],
+      "module": "amd",
+      "target": "esnext",
+      "listFiles": true,
+      "listEmittedFiles": true,
+      // "types" : ["typescript.d.ts"],
+      "typeRoots" : ["$typeRoots$"],
+      // Emit the source alongside the sourcemaps within a single file;
+      // requires --inlineSourceMap or --sourceMap to be set.
+      // "inlineSources": true,
+      "sourceMap": true,
+      "outFile": bundle,
+    },
+  });
+
+  let mut root_names_str: Vec<String> = root_names
+    .iter()
+    .map(|p| {
+      if !p.exists() {
+        panic!("File not found {}", p.display());
+      }
+
+      let module_specifier =
+        ModuleSpecifier::resolve_url_or_path(&p.to_string_lossy()).unwrap();
+      module_specifier.as_str().to_string()
+    })
+    .collect();
+  root_names_str.push("$asset$/lib.deno_core.d.ts".to_string());
+
+  // TODO lift js_check to caller?
+  let state = js_check(ts_isolate.compile(&config_json, root_names_str));
+
+  Ok(state)
+}
+
+#[allow(dead_code)]
+fn print_source_code(code: &str) {
+  let mut i = 1;
+  for line in code.lines() {
+    println!("{:3}  {}", i, line);
+    i += 1;
+  }
+}
+
+/// Create a V8 snapshot.
+pub fn mksnapshot_bundle(
+  bundle: &Path,
+  state: Arc<Mutex<TSState>>,
+) -> Result<(), ErrBox> {
+  let mut runtime_isolate = Isolate::new(StartupData::None, true);
+  let source_code_vec = std::fs::read(bundle)?;
+  let source_code = std::str::from_utf8(&source_code_vec)?;
+
+  js_check(runtime_isolate.execute("amd_runtime.js", AMD_RUNTIME_CODE));
+  js_check(runtime_isolate.execute(&bundle.to_string_lossy(), &source_code));
+
+  let main = state.lock().unwrap().main_module_name();
+  js_check(runtime_isolate.execute("anon", &format!("require('{}')", main)));
+
+  write_snapshot(runtime_isolate, bundle)?;
+
+  Ok(())
+}
+
+/// Create a V8 snapshot. This differs from mksnapshot_bundle in that is also
+/// runs typescript.js
+pub fn mksnapshot_bundle_ts(
+  bundle: &Path,
+  state: Arc<Mutex<TSState>>,
+) -> Result<(), ErrBox> {
+  let mut runtime_isolate = Isolate::new(StartupData::None, true);
+  let source_code_vec = std::fs::read(bundle)?;
+  let source_code = std::str::from_utf8(&source_code_vec)?;
+
+  js_check(runtime_isolate.execute("amd_runtime.js", AMD_RUNTIME_CODE));
+  js_check(runtime_isolate.execute("typescript.js", TYPESCRIPT_CODE));
+  js_check(runtime_isolate.execute(&bundle.to_string_lossy(), &source_code));
+
+  let main = state.lock().unwrap().main_module_name();
+  js_check(runtime_isolate.execute("anon", &format!("require('{}')", main)));
+
+  write_snapshot(runtime_isolate, bundle)?;
+
+  Ok(())
+}
+
+fn write_snapshot(
+  runtime_isolate: Isolate,
+  bundle: &Path,
+) -> Result<(), ErrBox> {
+  println!("creating snapshot...");
+  let snapshot = runtime_isolate.snapshot()?;
+  let snapshot_slice =
+    unsafe { std::slice::from_raw_parts(snapshot.data_ptr, snapshot.data_len) };
+  println!("snapshot bytes {}", snapshot_slice.len());
+
+  let snapshot_path = bundle.with_extension("bin");
+
+  fs::write(&snapshot_path, snapshot_slice)?;
+  println!("snapshot path {} ", snapshot_path.display());
+  Ok(())
+}
+
+macro_rules! inc {
+  ($e:expr) => {
+    Some(include_str!(concat!(
+      "../third_party/node_modules/typescript/lib/",
+      $e
+    )))
+  };
+}
+
+/// Same as get_asset() but returns NotFound intead of None.
+pub fn get_asset2(name: &str) -> Result<&'static str, ErrBox> {
+  match get_asset(name) {
+    Some(a) => Ok(a),
+    None => Err(
+      std::io::Error::new(std::io::ErrorKind::NotFound, "Asset not found")
+        .into(),
+    ),
+  }
+}
+
+pub fn get_asset(name: &str) -> Option<&'static str> {
+  match name {
+    "lib.deno_core.d.ts" => Some(include_str!("lib.deno_core.d.ts")),
+    "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"),
+    "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"),
+    "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"),
+    "lib.es2017.d.ts" => inc!("lib.es2017.d.ts"),
+    "lib.es2016.d.ts" => inc!("lib.es2016.d.ts"),
+    "lib.es5.d.ts" => inc!("lib.es5.d.ts"),
+    "lib.es2015.d.ts" => inc!("lib.es2015.d.ts"),
+    "lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"),
+    "lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"),
+    "lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"),
+    "lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"),
+    "lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"),
+    "lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"),
+    "lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"),
+    "lib.es2015.symbol.wellknown.d.ts" => {
+      inc!("lib.es2015.symbol.wellknown.d.ts")
+    }
+    "lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"),
+    "lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"),
+    "lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"),
+    "lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"),
+    "lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"),
+    "lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"),
+    "lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"),
+    "lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"),
+    "lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"),
+    "lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"),
+    "lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"),
+    "lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"),
+    "lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"),
+    "lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"),
+    "lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"),
+    "lib.esnext.bigint.d.ts" => inc!("lib.esnext.bigint.d.ts"),
+    "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"),
+    _ => None,
+  }
+}
+
+/// Sets the --trace-serializer V8 flag for debugging snapshots.
+pub fn trace_serializer() {
+  let dummy = "foo".to_string();
+  let r =
+    deno::v8_set_flags(vec![dummy.clone(), "--trace-serializer".to_string()]);
+  assert_eq!(r, vec![dummy]);
+}
+
+fn preprocessor_replacements_json() -> String {
+  /// BUILD_OS and BUILD_ARCH match the values in Deno.build. See js/build.ts.
+  #[cfg(target_os = "macos")]
+  static BUILD_OS: &str = "mac";
+  #[cfg(target_os = "linux")]
+  static BUILD_OS: &str = "linux";
+  #[cfg(target_os = "windows")]
+  static BUILD_OS: &str = "win";
+  #[cfg(target_arch = "x86_64")]
+  static BUILD_ARCH: &str = "x64";
+
+  let mut replacements = HashMap::new();
+  replacements.insert("DENO_REPLACE_OS", BUILD_OS);
+  replacements.insert("DENO_REPLACE_ARCH", BUILD_ARCH);
+  serde_json::json!(replacements).to_string()
+}

deno_typescript/ops.rs

@@ -0,0 +1,141 @@
+use crate::TSState;
+use deno::CoreOp;
+use deno::ErrBox;
+use deno::ModuleSpecifier;
+use deno::Op;
+use deno::OpId;
+use serde::Deserialize;
+use serde_json::json;
+use serde_json::Value;
+
+#[derive(Debug)]
+pub struct WrittenFile {
+  pub url: String,
+  pub module_name: String,
+  pub source_code: String,
+}
+
+fn dispatch2(
+  s: &mut TSState,
+  op_id: OpId,
+  control_buf: &[u8],
+) -> Result<Value, ErrBox> {
+  let v = serde_json::from_slice(control_buf)?;
+  // Warning! The op_id values below are shared between this code and
+  // compiler_main.js. Update with care!
+  match op_id {
+    49 => read_file(s, v),
+    50 => exit(s, v),
+    51 => write_file(s, v),
+    52 => resolve_module_names(s, v),
+    53 => set_emit_result(s, v),
+    _ => unreachable!(),
+  }
+}
+
+pub fn dispatch_op(s: &mut TSState, op_id: OpId, control_buf: &[u8]) -> CoreOp {
+  let result = dispatch2(s, op_id, control_buf);
+  let response = match result {
+    Ok(v) => json!({ "ok": v }),
+    Err(err) => json!({ "err": err.to_string() }),
+  };
+  let x = serde_json::to_string(&response).unwrap();
+  let vec = x.into_bytes();
+  Op::Sync(vec.into_boxed_slice())
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ReadFile {
+  file_name: String,
+  language_version: Option<i32>,
+  should_create_new_source_file: bool,
+}
+
+fn read_file(_s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let v: ReadFile = serde_json::from_value(v)?;
+  let (module_name, source_code) = if v.file_name.starts_with("$asset$/") {
+    let asset = v.file_name.replace("$asset$/", "");
+    let source_code = crate::get_asset2(&asset)?.to_string();
+    (asset, source_code)
+  } else {
+    assert!(!v.file_name.starts_with("$assets$"), "you meant $asset$");
+    let module_specifier = ModuleSpecifier::resolve_url_or_path(&v.file_name)?;
+    let path = module_specifier.as_url().to_file_path().unwrap();
+    println!("cargo:rerun-if-changed={}", path.display());
+    (
+      module_specifier.as_str().to_string(),
+      std::fs::read_to_string(&path)?,
+    )
+  };
+  Ok(json!({
+    "moduleName": module_name,
+    "sourceCode": source_code,
+  }))
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct WriteFile {
+  file_name: String,
+  data: String,
+  module_name: String,
+}
+
+fn write_file(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let v: WriteFile = serde_json::from_value(v)?;
+  let module_specifier = ModuleSpecifier::resolve_url_or_path(&v.file_name)?;
+  if s.bundle {
+    std::fs::write(&v.file_name, &v.data)?;
+  }
+  s.written_files.push(WrittenFile {
+    url: module_specifier.as_str().to_string(),
+    module_name: v.module_name,
+    source_code: v.data,
+  });
+  Ok(json!(true))
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct ResolveModuleNames {
+  module_names: Vec<String>,
+  containing_file: String,
+}
+
+fn resolve_module_names(_s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let v: ResolveModuleNames = serde_json::from_value(v).unwrap();
+  let mut resolved = Vec::<String>::new();
+  let referrer = ModuleSpecifier::resolve_url_or_path(&v.containing_file)?;
+  for specifier in v.module_names {
+    let ms = ModuleSpecifier::resolve_import(&specifier, referrer.as_str())?;
+    resolved.push(ms.as_str().to_string());
+  }
+  Ok(json!(resolved))
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct Exit {
+  code: i32,
+}
+
+fn exit(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let v: Exit = serde_json::from_value(v)?;
+  s.exit_code = v.code;
+  std::process::exit(v.code)
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct EmitResult {
+  pub emit_skipped: bool,
+  pub diagnostics: Vec<String>,
+  pub emitted_files: Vec<String>,
+}
+
+fn set_emit_result(s: &mut TSState, v: Value) -> Result<Value, ErrBox> {
+  let v: EmitResult = serde_json::from_value(v)?;
+  s.emit_result = Some(v);
+  Ok(json!(true))
+}