Make bundles fully standalone (#3325)

- Bundles are fully standalone. They now include the shared loader with
  `deno_typescript`.
- Refactor of the loader in `deno_typescript` to perform module
  instantiation in a more
- Change of behaviour when an output file is not specified on the CLI.
  Previously a default name was determined and the bundle written to that
  file, now the bundle will be sent to `stdout`.
- Refactors in the TypeScript compiler to be able to support the concept
  of a request type.  This provides a cleaner abstraction and makes it
  easier to support things like single module transpiles to the userland.
- Remove a "dangerous" circular dependency between `os.ts` and `deno.ts`,
  and define `pid` and `noColor` in a better way.
- Don't bind early to `console` in `repl.ts`.
- Add an integration test for generating a bundle.

cli/js/compiler.ts

@@ -31,6 +31,13 @@ enum MediaType {
   Unknown = 5
 }
 
+// Warning! The values in this enum are duplicated in cli/msg.rs
+// Update carefully!
+enum CompilerRequestType {
+  Compile = 0,
+  Bundle = 1
+}
+
 // Startup boilerplate. This is necessary because the compiler has its own
 // snapshot. (It would be great if we could remove these things or centralize
 // them somewhere else.)
@@ -44,16 +51,23 @@ window["denoMain"] = denoMain;
 
 const ASSETS = "$asset$";
 const OUT_DIR = "$deno$";
+const BUNDLE_LOADER = "bundle_loader.js";
 
 /** The format of the work message payload coming from the privileged side */
-interface CompilerReq {
+type CompilerRequest = {
   rootNames: string[];
-  bundle?: string;
   // TODO(ry) add compiler config to this interface.
   // options: ts.CompilerOptions;
   configPath?: string;
   config?: string;
-}
+} & (
+  | {
+      type: CompilerRequestType.Compile;
+    }
+  | {
+      type: CompilerRequestType.Bundle;
+      outFile?: string;
+    });
 
 interface ConfigureResponse {
   ignoredOptions?: string[];
@@ -271,7 +285,7 @@ function fetchSourceFiles(
 async function processImports(
   specifiers: Array<[string, string]>,
   referrer = ""
-): Promise<void> {
+): Promise<SourceFileJson[]> {
   if (!specifiers.length) {
     return;
   }
@@ -287,6 +301,7 @@ async function processImports(
       await processImports(sourceFile.imports(), sourceFile.url);
     }
   }
+  return sourceFiles;
 }
 
 /** Utility function to turn the number of bytes into a human readable
@@ -314,16 +329,36 @@ function cache(extension: string, moduleId: string, contents: string): void {
 const encoder = new TextEncoder();
 
 /** Given a fileName and the data, emit the file to the file system. */
-function emitBundle(fileName: string, data: string): void {
+function emitBundle(
+  rootNames: string[],
+  fileName: string | undefined,
+  data: string,
+  sourceFiles: readonly ts.SourceFile[]
+): void {
   // For internal purposes, when trying to emit to `$deno$` just no-op
-  if (fileName.startsWith("$deno$")) {
+  if (fileName && fileName.startsWith("$deno$")) {
     console.warn("skipping emitBundle", fileName);
     return;
   }
-  const encodedData = encoder.encode(data);
-  console.log(`Emitting bundle to "${fileName}"`);
-  writeFileSync(fileName, encodedData);
-  console.log(`${humanFileSize(encodedData.length)} emitted.`);
+  const loader = fetchAsset(BUNDLE_LOADER);
+  // when outputting to AMD and a single outfile, TypeScript makes up the module
+  // specifiers which are used to define the modules, and doesn't expose them
+  // publicly, so we have to try to replicate
+  const sources = sourceFiles.map(sf => sf.fileName);
+  const sharedPath = util.commonPath(sources);
+  rootNames = rootNames.map(id =>
+    id.replace(sharedPath, "").replace(/\.\w+$/i, "")
+  );
+  const instantiate = `instantiate(${JSON.stringify(rootNames)});\n`;
+  const bundle = `${loader}\n${data}\n${instantiate}`;
+  if (fileName) {
+    const encodedData = encoder.encode(bundle);
+    console.warn(`Emitting bundle to "${fileName}"`);
+    writeFileSync(fileName, encodedData);
+    console.warn(`${humanFileSize(encodedData.length)} emitted.`);
+  } else {
+    console.log(bundle);
+  }
 }
 
 /** Returns the TypeScript Extension enum for a given media type. */
@@ -380,17 +415,23 @@ class Host implements ts.CompilerHost {
 
   /** Provides the `ts.HostCompiler` interface for Deno.
    *
+   * @param _rootNames A set of modules that are the ones that should be
+   *   instantiated first.  Used when generating a bundle.
    * @param _bundle Set to a string value to configure the host to write out a
    *   bundle instead of caching individual files.
    */
-  constructor(private _bundle?: string) {
-    if (this._bundle) {
+  constructor(
+    private _requestType: CompilerRequestType,
+    private _rootNames: string[],
+    private _outFile?: string
+  ) {
+    if (this._requestType === CompilerRequestType.Bundle) {
       // options we need to change when we are generating a bundle
       const bundlerOptions: ts.CompilerOptions = {
         module: ts.ModuleKind.AMD,
-        inlineSourceMap: true,
         outDir: undefined,
         outFile: `${OUT_DIR}/bundle.js`,
+        // disabled until we have effective way to modify source maps
         sourceMap: false
       };
       Object.assign(this._options, bundlerOptions);
@@ -531,10 +572,11 @@ class Host implements ts.CompilerHost {
   ): void {
     util.log("compiler::host.writeFile", fileName);
     try {
-      if (this._bundle) {
-        emitBundle(this._bundle, data);
+      assert(sourceFiles != null);
+      if (this._requestType === CompilerRequestType.Bundle) {
+        emitBundle(this._rootNames, this._outFile, data, sourceFiles!);
       } else {
-        assert(sourceFiles != null && sourceFiles.length == 1);
+        assert(sourceFiles.length == 1);
         const url = sourceFiles![0].fileName;
         const sourceFile = SourceFile.get(url);
 
@@ -579,16 +621,29 @@ class Host implements ts.CompilerHost {
 // lazy instantiating the compiler web worker
 window.compilerMain = function compilerMain(): void {
   // workerMain should have already been called since a compiler is a worker.
-  window.onmessage = async ({ data }: { data: CompilerReq }): Promise<void> => {
-    const { rootNames, configPath, config, bundle } = data;
-    util.log(">>> compile start", { rootNames, bundle });
+  window.onmessage = async ({
+    data: request
+  }: {
+    data: CompilerRequest;
+  }): Promise<void> => {
+    const { rootNames, configPath, config } = request;
+    util.log(">>> compile start", {
+      rootNames,
+      type: CompilerRequestType[request.type]
+    });
 
     // This will recursively analyse all the code for other imports, requesting
     // those from the privileged side, populating the in memory cache which
     // will be used by the host, before resolving.
-    await processImports(rootNames.map(rootName => [rootName, rootName]));
+    const resolvedRootModules = (await processImports(
+      rootNames.map(rootName => [rootName, rootName])
+    )).map(info => info.url);
 
-    const host = new Host(bundle);
+    const host = new Host(
+      request.type,
+      resolvedRootModules,
+      request.type === CompilerRequestType.Bundle ? request.outFile : undefined
+    );
     let emitSkipped = true;
     let diagnostics: ts.Diagnostic[] | undefined;
 
@@ -642,8 +697,9 @@ window.compilerMain = function compilerMain(): void {
 
       // We will only proceed with the emit if there are no diagnostics.
       if (diagnostics && diagnostics.length === 0) {
-        if (bundle) {
-          console.log(`Bundling "${bundle}"`);
+        if (request.type === CompilerRequestType.Bundle) {
+          // warning so it goes to stderr instead of stdout
+          console.warn(`Bundling "${resolvedRootModules.join(`", "`)}"`);
         }
         const emitResult = program.emit();
         emitSkipped = emitResult.emitSkipped;
@@ -662,7 +718,10 @@ window.compilerMain = function compilerMain(): void {
 
     postMessage(result);
 
-    util.log("<<< compile end", { rootNames, bundle });
+    util.log("<<< compile end", {
+      rootNames,
+      type: CompilerRequestType[request.type]
+    });
 
     // The compiler isolate exits after a single message.
     workerClose();

cli/js/deno.ts

@@ -112,9 +112,3 @@ export let pid: number;
 
 /** Reflects the NO_COLOR environment variable: https://no-color.org/ */
 export let noColor: boolean;
-
-// TODO(ry) This should not be exposed to Deno.
-export function _setGlobals(pid_: number, noColor_: boolean): void {
-  pid = pid_;
-  noColor = noColor_;
-}

cli/js/os.ts

@@ -7,9 +7,6 @@ import * as util from "./util.ts";
 import { window } from "./window.ts";
 import { OperatingSystem, Arch } from "./build.ts";
 
-// builtin modules
-import { _setGlobals } from "./deno.ts";
-
 /** Check if running in terminal.
  *
  *       console.log(Deno.isTTY().stdout);
@@ -103,14 +100,15 @@ export function start(preserveDenoNamespace = true, source?: string): Start {
   // First we send an empty `Start` message to let the privileged side know we
   // are ready. The response should be a `StartRes` message containing the CLI
   // args and other info.
-  const s = sendSync(dispatch.OP_START);
+  const startResponse = sendSync(dispatch.OP_START);
+  const { pid, noColor, debugFlag } = startResponse;
 
-  util.setLogDebug(s.debugFlag, source);
+  util.setLogDebug(debugFlag, source);
 
   // pid and noColor need to be set in the Deno module before it's set to be
   // frozen.
-  _setGlobals(s.pid, s.noColor);
-  delete window.Deno._setGlobals;
+  util.immutableDefine(window.Deno, "pid", pid);
+  util.immutableDefine(window.Deno, "noColor", noColor);
   Object.freeze(window.Deno);
 
   if (preserveDenoNamespace) {
@@ -126,7 +124,7 @@ export function start(preserveDenoNamespace = true, source?: string): Start {
     assert(window.Deno === undefined);
   }
 
-  return s;
+  return startResponse;
 }
 
 /**

cli/js/repl.ts

@@ -8,8 +8,6 @@ import { stringifyArgs } from "./console.ts";
 import * as dispatch from "./dispatch.ts";
 import { sendSync, sendAsync } from "./dispatch_json.ts";
 
-const { console } = window;
-
 /**
  * REPL logging.
  * In favor of console.log to avoid unwanted indentation
@@ -106,6 +104,7 @@ function evaluate(code: string): boolean {
 
 // @internal
 export async function replLoop(): Promise<void> {
+  const { console } = window;
   Object.defineProperties(window, replCommands);
 
   const historyFile = "deno_history.txt";

cli/js/util.ts

@@ -223,3 +223,33 @@ export function splitNumberToParts(n: number): number[] {
   const higher = (n - lower) / 0x100000000;
   return [lower, higher];
 }
+
+/** Return the common path shared by the `paths`.
+ *
+ * @param paths The set of paths to compare.
+ * @param sep An optional separator to use. Defaults to `/`.
+ * @internal
+ */
+export function commonPath(paths: string[], sep = "/"): string {
+  const [first = "", ...remaining] = paths;
+  if (first === "" || remaining.length === 0) {
+    return "";
+  }
+  const parts = first.split(sep);
+
+  let endOfPrefix = parts.length;
+  for (const path of remaining) {
+    const compare = path.split(sep);
+    for (let i = 0; i < endOfPrefix; i++) {
+      if (compare[i] !== parts[i]) {
+        endOfPrefix = i;
+      }
+    }
+
+    if (endOfPrefix === 0) {
+      return "";
+    }
+  }
+  const prefix = parts.slice(0, endOfPrefix).join(sep);
+  return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`;
+}