Migrate internal bundles to System (#4233)

cli/js/compiler_api_test.ts

@@ -109,14 +109,14 @@ test(async function bundleApiSources() {
     "/bar.ts": `export const bar = "bar";\n`
   });
   assert(diagnostics == null);
-  assert(actual.includes(`__inst_s("foo")`));
+  assert(actual.includes(`__instantiate("foo")`));
   assert(actual.includes(`__exp["bar"]`));
 });
 
 test(async function bundleApiNoSources() {
   const [diagnostics, actual] = await bundle("./cli/tests/subdir/mod1.ts");
   assert(diagnostics == null);
-  assert(actual.includes(`__inst_s("mod1")`));
+  assert(actual.includes(`__instantiate("mod1")`));
   assert(actual.includes(`__exp["printHello3"]`));
 });
 

cli/js/compiler_bundler.ts

@@ -50,8 +50,8 @@ export function buildBundle(
   let instantiate: string;
   if (rootExports && rootExports.length) {
     instantiate = hasTla
-      ? `const __exp = await __inst("${rootName}");\n`
+      ? `const __exp = await __instantiateAsync("${rootName}");\n`
-      : `const __exp = __inst_s("${rootName}");\n`;
+      : `const __exp = __instantiate("${rootName}");\n`;
     for (const rootExport of rootExports) {
       if (rootExport === "default") {
         instantiate += `export default __exp["${rootExport}"];\n`;
@@ -61,8 +61,8 @@ export function buildBundle(
     }
   } else {
     instantiate = hasTla
-      ? `await __inst("${rootName}");\n`
+      ? `await __instantiateAsync("${rootName}");\n`
-      : `__inst_s("${rootName}");\n`;
+      : `__instantiate("${rootName}");\n`;
   }
   return `${SYSTEM_LOADER}\n${data}\n${instantiate}`;
 }

cli/js/runtime_main.ts

@@ -7,6 +7,9 @@
 //  - `bootstrapMainRuntime` - must be called once, when Isolate is created.
 //   It sets up runtime by providing globals for `WindowScope` and adds `Deno` global.
 
+import * as Deno from "./deno.ts";
+import * as domTypes from "./dom_types.ts";
+import * as csprng from "./get_random_values.ts";
 import {
   readOnly,
   writable,
@@ -14,21 +17,18 @@ import {
   windowOrWorkerGlobalScopeProperties,
   eventTargetProperties
 } from "./globals.ts";
-import * as domTypes from "./dom_types.ts";
-import { log } from "./util.ts";
-import * as runtime from "./runtime.ts";
-import { args } from "./deno.ts";
-import * as csprng from "./get_random_values.ts";
-import { replLoop } from "./repl.ts";
-import { setSignals } from "./process.ts";
-import * as Deno from "./deno.ts";
 import { internalObject } from "./internals.ts";
+import { setSignals } from "./process.ts";
+import { replLoop } from "./repl.ts";
+import * as runtime from "./runtime.ts";
+import { symbols } from "./symbols.ts";
+import { log } from "./util.ts";
 
 // TODO: factor out `Deno` global assignment to separate function
 // Add internal object to Deno object.
 // This is not exposed as part of the Deno types.
 // @ts-ignore
-Deno[Deno.symbols.internal] = internalObject;
+Deno[symbols.internal] = internalObject;
 
 export const mainRuntimeGlobalProperties = {
   window: readOnly(globalThis),
@@ -74,10 +74,10 @@ export function bootstrapMainRuntime(): void {
 
   log("cwd", s.cwd);
   for (let i = 0; i < s.args.length; i++) {
-    args.push(s.args[i]);
+    Deno.args.push(s.args[i]);
   }
-  log("args", args);
+  log("args", Deno.args);
-  Object.freeze(args);
+  Object.freeze(Deno.args);
 
   if (s.repl) {
     replLoop();

cli/source_maps.rs

@@ -389,7 +389,7 @@ mod tests {
     assert_eq!(actual.message, "TypeError: baz");
     // Because this is accessing the live bundle, this test might be more fragile
     assert_eq!(actual.frames.len(), 1);
-    assert!(actual.frames[0].script_name.ends_with("/dom_types.ts"));
+    assert_eq!(actual.frames[0].script_name, "$deno$/io.ts");
   }
 
   #[test]

cli/tests/bundle.test.out

@@ -1,5 +1,5 @@
 [WILDCARD]
-let System, __inst, __inst_s;
+let System, __instantiateAsync, __instantiate;
 [WILDCARD]
 (() => {
 [WILDCARD]
@@ -15,7 +15,7 @@ System.register("mod1", ["subdir2/mod2"], function (exports_3, context_3) {
 [WILDCARD]
 });
 
-const __exp = __inst_s("mod1");
+const __exp = __instantiate("mod1");
 export const returnsHi = __exp["returnsHi"];
 export const returnsFoo2 = __exp["returnsFoo2"];
 export const printHello3 = __exp["printHello3"];

deno_typescript/README.md

@@ -5,3 +5,62 @@
 
 This crate provides utilities to compile typescript, bundle it up, and create a
 V8 snapshot, all during build. Snapshots allow the executable to startup fast.
+
+## `system_loader.js`
+
+This is a minimalistic implementation of a
+[System](https://github.com/systemjs/systemjs) module loader. It is specifically
+designed to load modules that are emitted from TypeScript the module format is
+`"system"` and a single `"outfile"` is supplied, which is commonly refereed to
+as a bundle.
+
+Because this loader becomes part of an emitted bundle under `Deno.bundle()` and
+`deno bundle`, it has minimal comments and very terse and cryptic syntax, which
+isn't very self documenting. Because of this, a guide to this file is provided
+here.
+
+A bundle of System modules expects a `System.register()` function to be in scope
+for registering the modules. Modules that are emitted from TypeScript in a
+single out file always pass 3 arguments, the module specifier, an array of
+strings of modules specifiers that this module depends upon, and finally a
+module factory.
+
+The module factory requires two arguments to be passed, a function for exporting
+values and a context object. We have to bind to some information in the
+environment to provide these, so `gC` gets the context and `gE` gets the export
+function to be passed to a factory. The context contains information like the
+module specifier, a reference to the dynamic `import()` and the equivalent of
+`import.meta`. The export function takes either two arguments of an named export
+and its value, or an object record of keys of the named exports and the values
+of the exports.
+
+The running of the factories is handled by `rF()`. When the factory is run, it
+returns an object with two keys, `execute` and `setters`. `execute` is a
+function which finalises that instantiation of the module, and `setters` which
+is an array of functions that sets the value of the exports of the dependent
+module.
+
+The `gExp()` and `gExpA()` are the recursive functions which returns the exports
+of a given module. It will determine if the module has been fully initialized,
+and if not, it will gather the exports of the dependencies, set those exports in
+the module via the `setters` and run the modules `execute()`. It will then
+always return or resolve with the exports of the module.
+
+As of TypeScript 3.8, top level await is supported when emitting ES or System
+modules. When Deno creates a module bundle, it creates a valid, self-contained
+ES module which exports the exports of the "main" module that was used when the
+bundle was created. If a module in the bundle requires top-level-await, then the
+`execute()` function is emitted as an async function, returning a promise. This
+means that in order to export the values of the main module, the instantiation
+needs to utilise top-level-await as well.
+
+At the time of this writing, while V8 and other JavaScript engines have
+implemented top-level-await, no browsers have it implemented, meaning that most
+browsers could not consume modules that require top-level-await.
+
+In order to facilitate this, there are two functions that are in the scope of
+the module in addition to the `System.register()` method. `__instantiate(main)`
+will bootstrap everything synchronously and `__instantiate(main)` will do so
+asynchronously. When emitting a bundle that contains a module that requires
+top-level-await, Deno will detect this and utilise
+`await __instantiateAsync(main)` instead.

deno_typescript/bundle_loader.js

@@ -1,121 +0,0 @@
-// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-
-// A script preamble that provides the ability to load a single outfile
-// TypeScript "bundle" where a main module is loaded which recursively
-// instantiates all the other modules in the bundle.  This code is used to load
-// bundles when creating snapshots, but is also used when emitting bundles from
-// Deno cli.
-
-// @ts-nocheck
-
-/**
- * @type {(name: string, deps: ReadonlyArray<string>, factory: (...deps: any[]) => void) => void=}
- */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-let define;
-
-/**
- * @type {(mod: string) => any=}
- */
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-let instantiate;
-
-/**
- * @callback Factory
- * @argument {...any[]} args
- * @returns {object | void}
- */
-
-/**
- * @typedef ModuleMetaData
- * @property {ReadonlyArray<string>} dependencies
- * @property {(Factory | object)=} factory
- * @property {object} exports
- */
-
-(function() {
-  /**
-   * @type {Map<string, ModuleMetaData>}
-   */
-  const modules = new Map();
-
-  /**
-   * Bundles in theory can support "dynamic" imports, but for internal bundles
-   * we can't go outside to fetch any modules that haven't been statically
-   * defined.
-   * @param {string[]} deps
-   * @param {(...deps: any[]) => void} resolve
-   * @param {(err: any) => void} reject
-   */
-  const require = (deps, resolve, reject) => {
-    try {
-      if (deps.length !== 1) {
-        throw new TypeError("Expected only a single module specifier.");
-      }
-      if (!modules.has(deps[0])) {
-        throw new RangeError(`Module "${deps[0]}" not defined.`);
-      }
-      resolve(getExports(deps[0]));
-    } catch (e) {
-      if (reject) {
-        reject(e);
-      } else {
-        throw e;
-      }
-    }
-  };
-
-  define = (id, dependencies, factory) => {
-    if (modules.has(id)) {
-      throw new RangeError(`Module "${id}" has already been defined.`);
-    }
-    modules.set(id, {
-      dependencies,
-      factory,
-      exports: {}
-    });
-  };
-
-  /**
-   * @param {string} id
-   * @returns {any}
-   */
-  function getExports(id) {
-    const module = modules.get(id);
-    if (!module) {
-      // because `$deno$/ts_global.d.ts` looks like a real script, it doesn't
-      // get erased from output as an import, but it doesn't get defined, so
-      // we don't have a cache for it, so because this is an internal bundle
-      // we can just safely return an empty object literal.
-      return {};
-    }
-    if (!module.factory) {
-      return module.exports;
-    } else if (module.factory) {
-      const { factory, exports } = module;
-      delete module.factory;
-      if (typeof factory === "function") {
-        const dependencies = module.dependencies.map(id => {
-          if (id === "require") {
-            return require;
-          } else if (id === "exports") {
-            return exports;
-          }
-          return getExports(id);
-        });
-        factory(...dependencies);
-      } else {
-        Object.assign(exports, factory);
-      }
-      return exports;
-    }
-  }
-
-  instantiate = dep => {
-    define = undefined;
-    const result = getExports(dep);
-    // clean up, or otherwise these end up in the runtime environment
-    instantiate = undefined;
-    return result;
-  };
-})();

deno_typescript/lib.rs

@@ -25,7 +25,7 @@ use std::sync::Mutex;
 
 static TYPESCRIPT_CODE: &str = include_str!("typescript/lib/typescript.js");
 static COMPILER_CODE: &str = include_str!("compiler_main.js");
-static BUNDLE_LOADER: &str = include_str!("bundle_loader.js");
+static SYSTEM_LOADER: &str = include_str!("system_loader.js");
 
 pub fn ts_version() -> String {
   let data = include_str!("typescript/package.json");
@@ -143,20 +143,20 @@ pub fn compile_bundle(
 
   let config_json = serde_json::json!({
     "compilerOptions": {
-      "strict": true,
       "declaration": true,
-      "lib": ["esnext"],
-      "module": "amd",
-      "target": "esnext",
-      "listFiles": true,
-      "listEmittedFiles": true,
-      // "types" : ["typescript.d.ts"],
-      "typeRoots" : ["$typeRoots$"],
       // Emit the source alongside the sourcemaps within a single file;
       // requires --inlineSourceMap or --sourceMap to be set.
       // "inlineSources": true,
-      "sourceMap": true,
+      "lib": ["esnext"],
+      "listEmittedFiles": true,
+      "listFiles": true,
+      "module": "system",
       "outFile": bundle_filename,
+      "removeComments": true,
+      "sourceMap": true,
+      "strict": true,
+      "target": "esnext",
+      "typeRoots" : ["$typeRoots$"],
     },
   });
 
@@ -198,13 +198,13 @@ pub fn mksnapshot_bundle(
   bundle_filename: &Path,
   main_module_name: &str,
 ) -> Result<(), ErrBox> {
-  js_check(isolate.execute("bundle_loader.js", BUNDLE_LOADER));
+  js_check(isolate.execute("system_loader.js", SYSTEM_LOADER));
   let source_code_vec = std::fs::read(bundle_filename).unwrap();
   let bundle_source_code = std::str::from_utf8(&source_code_vec).unwrap();
   js_check(
     isolate.execute(&bundle_filename.to_string_lossy(), bundle_source_code),
   );
-  let script = &format!("instantiate('{}')", main_module_name);
+  let script = &format!("__instantiate(\"{}\");", main_module_name);
   js_check(isolate.execute("anon", script));
   write_snapshot(isolate, snapshot_filename)?;
   Ok(())

deno_typescript/system_loader.js

@@ -4,30 +4,29 @@
 
 // @ts-nocheck
 /* eslint-disable */
-
+let System, __instantiateAsync, __instantiate;
-let System, __inst, __inst_s;
 
 (() => {
-  const mMap = new Map();
+  const r = new Map();
+
   System = {
     register(id, d, f) {
-      mMap.set(id, { id, d, f, exp: {} });
+      r.set(id, { d, f, exp: {} });
     }
   };
 
-  const gC = (data, main) => {
+  function gC(id, main) {
-    const { id } = data;
     return {
       id,
-      import: async id => mMap.get(id)?.exp,
+      import: async id => r.get(id)?.exp,
       meta: { url: id, main }
     };
-  };
+  }
 
-  const gE = ({ exp }) => {
+  function gE(exp) {
     return (id, v) => {
-      const vs = typeof id === "string" ? { [id]: v } : id;
+      v = typeof id === "string" ? { [id]: v } : id;
-      for (const [id, value] of Object.entries(vs)) {
+      for (const [id, value] of Object.entries(v)) {
         Object.defineProperty(exp, id, {
           value,
           writable: true,
@@ -35,65 +34,54 @@ let System, __inst, __inst_s;
         });
       }
     };
-  };
+  }
 
-  const iQ = [];
+  function rF(main) {
-
+    for (const [id, m] of r.entries()) {
-  const enq = ids => {
+      const { f, exp } = m;
-    for (const id of ids) {
+      const { execute: e, setters: s } = f(gE(exp), gC(id, id === main));
-      if (!iQ.includes(id)) {
-        const { d } = mMap.get(id);
-        iQ.push(id);
-        enq(d);
-      }
-    }
-  };
-
-  const gRQ = main => {
-    const rQ = [];
-    let id;
-    while ((id = iQ.pop())) {
-      const m = mMap.get(id),
-        { f } = m;
-      if (!f) return;
-      rQ.push([m.d, f(gE(m), gC(m, id === main))]);
       delete m.f;
+      m.e = e;
+      m.s = s;
     }
-    return rQ;
+  }
-  };
 
-  const dr = async main => {
+  async function gExpA(id) {
-    const rQ = gRQ(main);
+    if (!r.has(id)) return;
-    let r;
+    const m = r.get(id);
-    while ((r = rQ.shift())) {
+    if (m.s) {
-      const [d, { execute, setters }] = r;
+      const { d, e, s } = m;
-      for (let i = 0; i < d.length; i++) setters[i](mMap.get(d[i])?.exp);
+      delete m.s;
-      const e = execute();
+      delete m.e;
-      if (e) await e;
+      for (let i = 0; i < s.length; i++) s[i](await gExpA(d[i]));
+      const r = e();
+      if (r) await r;
     }
-  };
+    return m.exp;
+  }
 
-  const dr_s = main => {
+  function gExp(id) {
-    const rQ = gRQ(main);
+    if (!r.has(id)) return;
-    let r;
+    const m = r.get(id);
-    while ((r = rQ.shift())) {
+    if (m.s) {
-      const [d, { execute, setters }] = r;
+      const { d, e, s } = m;
-      for (let i = 0; i < d.length; i++) setters[i](mMap.get(d[i])?.exp);
+      delete m.s;
-      execute();
+      delete m.e;
+      for (let i = 0; i < s.length; i++) s[i](gExp(d[i]));
+      e();
     }
+    return m.exp;
+  }
+
+  __instantiateAsync = async m => {
+    System = __instantiateAsync = __instantiate = undefined;
+    rF(m);
+    return gExpA(m);
   };
 
-  __inst = async id => {
+  __instantiate = m => {
-    System = __inst = __inst_s = undefined;
+    System = __instantiateAsync = __instantiate = undefined;
-    enq([id]);
+    rF(m);
-    await dr(id);
+    return gExp(m);
-    return mMap.get(id)?.exp;
-  };
-
-  __inst_s = id => {
-    System = __inst = __inst_s = undefined;
-    enq([id]);
-    dr_s(id);
-    return mMap.get(id)?.exp;
   };
 })();