mirror of
https://github.com/denoland/deno.git
synced 2025-09-26 12:19:12 +00:00
refactor(cli): migrate runtime compile/bundle to new infrastructure (#8192)
Fixes #8060
This commit is contained in:
Kitson Kelly
GitHub
parent
3558769d46
commit
fdcc78500c
23 changed files with 852 additions and 2770 deletions
|
|
@ -118,9 +118,6 @@ delete Object.prototype.__proto__;
|
|||
return core.decode(sourceCodeBytes);
|
||||
}
|
||||
|
||||
// Constants used by `normalizeString` and `resolvePath`
|
||||
const CHAR_DOT = 46; /* . */
|
||||
const CHAR_FORWARD_SLASH = 47; /* / */
|
||||
// Using incremental compile APIs requires that all
|
||||
// paths must be either relative or absolute. Since
|
||||
// analysis in Rust operates on fully resolved URLs,
|
||||
|
|
@ -218,18 +215,6 @@ delete Object.prototype.__proto__;
|
|||
*/
|
||||
const RESOLVED_SPECIFIER_CACHE = new Map();
|
||||
|
||||
function parseCompilerOptions(compilerOptions) {
|
||||
const { options, errors } = ts.convertCompilerOptionsFromJson(
|
||||
compilerOptions,
|
||||
"",
|
||||
"tsconfig.json",
|
||||
);
|
||||
return {
|
||||
options,
|
||||
diagnostics: errors.length ? errors : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
class SourceFile {
|
||||
constructor(json) {
|
||||
this.processed = false;
|
||||
|
|
@ -541,95 +526,6 @@ delete Object.prototype.__proto__;
|
|||
host,
|
||||
});
|
||||
|
||||
// This function is called only during snapshotting process
|
||||
const SYSTEM_LOADER = getAsset("system_loader.js");
|
||||
const SYSTEM_LOADER_ES5 = getAsset("system_loader_es5.js");
|
||||
|
||||
function buildLocalSourceFileCache(sourceFileMap) {
|
||||
for (const entry of Object.values(sourceFileMap)) {
|
||||
assert(entry.sourceCode.length > 0);
|
||||
SourceFile.addToCache({
|
||||
url: entry.url,
|
||||
filename: entry.url,
|
||||
mediaType: entry.mediaType,
|
||||
sourceCode: entry.sourceCode,
|
||||
versionHash: entry.versionHash,
|
||||
});
|
||||
|
||||
for (const importDesc of entry.imports) {
|
||||
let mappedUrl = importDesc.resolvedSpecifier;
|
||||
const importedFile = sourceFileMap[importDesc.resolvedSpecifier];
|
||||
assert(importedFile);
|
||||
const isJsOrJsx = importedFile.mediaType === MediaType.JavaScript ||
|
||||
importedFile.mediaType === MediaType.JSX;
|
||||
// If JS or JSX perform substitution for types if available
|
||||
if (isJsOrJsx) {
|
||||
// @deno-types has highest precedence, followed by
|
||||
// X-TypeScript-Types header
|
||||
if (importDesc.resolvedTypeDirective) {
|
||||
mappedUrl = importDesc.resolvedTypeDirective;
|
||||
} else if (importedFile.typeHeaders.length > 0) {
|
||||
const typeHeaders = importedFile.typeHeaders[0];
|
||||
mappedUrl = typeHeaders.resolvedSpecifier;
|
||||
} else if (importedFile.typesDirectives.length > 0) {
|
||||
const typeDirective = importedFile.typesDirectives[0];
|
||||
mappedUrl = typeDirective.resolvedSpecifier;
|
||||
}
|
||||
}
|
||||
|
||||
mappedUrl = mappedUrl.replace("memory://", "");
|
||||
SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url);
|
||||
}
|
||||
for (const fileRef of entry.referencedFiles) {
|
||||
SourceFile.cacheResolvedUrl(
|
||||
fileRef.resolvedSpecifier.replace("memory://", ""),
|
||||
fileRef.specifier,
|
||||
entry.url,
|
||||
);
|
||||
}
|
||||
for (const fileRef of entry.libDirectives) {
|
||||
SourceFile.cacheResolvedUrl(
|
||||
fileRef.resolvedSpecifier.replace("memory://", ""),
|
||||
fileRef.specifier,
|
||||
entry.url,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Warning! The values in this enum are duplicated in `cli/msg.rs`
|
||||
// Update carefully!
|
||||
const CompilerRequestType = {
|
||||
RuntimeCompile: 2,
|
||||
RuntimeBundle: 3,
|
||||
};
|
||||
|
||||
function createBundleWriteFile(state) {
|
||||
return function writeFile(_fileName, data, sourceFiles) {
|
||||
assert(sourceFiles != null);
|
||||
assert(state.options);
|
||||
// we only support single root names for bundles
|
||||
assert(state.rootNames.length === 1);
|
||||
state.bundleOutput = buildBundle(
|
||||
state.rootNames[0],
|
||||
data,
|
||||
sourceFiles,
|
||||
state.options.target ?? ts.ScriptTarget.ESNext,
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntimeCompileWriteFile(state) {
|
||||
return function writeFile(fileName, data, sourceFiles) {
|
||||
assert(sourceFiles);
|
||||
assert(sourceFiles.length === 1);
|
||||
state.emitMap[fileName] = {
|
||||
filename: sourceFiles[0].fileName,
|
||||
contents: data,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
const IGNORED_DIAGNOSTICS = [
|
||||
// TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is
|
||||
// not a module.
|
||||
|
|
@ -674,7 +570,6 @@ delete Object.prototype.__proto__;
|
|||
|
||||
function performanceStart() {
|
||||
stats.length = 0;
|
||||
// TODO(kitsonk) replace with performance.mark() when landed
|
||||
statsStart = new Date();
|
||||
ts.performance.enable();
|
||||
}
|
||||
|
|
@ -716,317 +611,6 @@ delete Object.prototype.__proto__;
|
|||
return stats;
|
||||
}
|
||||
|
||||
function normalizeString(path) {
|
||||
let res = "";
|
||||
let lastSegmentLength = 0;
|
||||
let lastSlash = -1;
|
||||
let dots = 0;
|
||||
let code;
|
||||
for (let i = 0, len = path.length; i <= len; ++i) {
|
||||
if (i < len) code = path.charCodeAt(i);
|
||||
else if (code === CHAR_FORWARD_SLASH) break;
|
||||
else code = CHAR_FORWARD_SLASH;
|
||||
|
||||
if (code === CHAR_FORWARD_SLASH) {
|
||||
if (lastSlash === i - 1 || dots === 1) {
|
||||
// NOOP
|
||||
} else if (lastSlash !== i - 1 && dots === 2) {
|
||||
if (
|
||||
res.length < 2 ||
|
||||
lastSegmentLength !== 2 ||
|
||||
res.charCodeAt(res.length - 1) !== CHAR_DOT ||
|
||||
res.charCodeAt(res.length - 2) !== CHAR_DOT
|
||||
) {
|
||||
if (res.length > 2) {
|
||||
const lastSlashIndex = res.lastIndexOf("/");
|
||||
if (lastSlashIndex === -1) {
|
||||
res = "";
|
||||
lastSegmentLength = 0;
|
||||
} else {
|
||||
res = res.slice(0, lastSlashIndex);
|
||||
lastSegmentLength = res.length - 1 - res.lastIndexOf("/");
|
||||
}
|
||||
lastSlash = i;
|
||||
dots = 0;
|
||||
continue;
|
||||
} else if (res.length === 2 || res.length === 1) {
|
||||
res = "";
|
||||
lastSegmentLength = 0;
|
||||
lastSlash = i;
|
||||
dots = 0;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (res.length > 0) res += "/" + path.slice(lastSlash + 1, i);
|
||||
else res = path.slice(lastSlash + 1, i);
|
||||
lastSegmentLength = i - lastSlash - 1;
|
||||
}
|
||||
lastSlash = i;
|
||||
dots = 0;
|
||||
} else if (code === CHAR_DOT && dots !== -1) {
|
||||
++dots;
|
||||
} else {
|
||||
dots = -1;
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
function commonPath(paths, sep = "/") {
|
||||
const [first = "", ...remaining] = paths;
|
||||
if (first === "" || remaining.length === 0) {
|
||||
return first.substring(0, first.lastIndexOf(sep) + 1);
|
||||
}
|
||||
const parts = first.split(sep);
|
||||
|
||||
let endOfPrefix = parts.length;
|
||||
for (const path of remaining) {
|
||||
const compare = path.split(sep);
|
||||
for (let i = 0; i < endOfPrefix; i++) {
|
||||
if (compare[i] !== parts[i]) {
|
||||
endOfPrefix = i;
|
||||
}
|
||||
}
|
||||
|
||||
if (endOfPrefix === 0) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
const prefix = parts.slice(0, endOfPrefix).join(sep);
|
||||
return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`;
|
||||
}
|
||||
|
||||
let rootExports;
|
||||
|
||||
function normalizeUrl(rootName) {
|
||||
const match = /^(\S+:\/{2,3})(.+)$/.exec(rootName);
|
||||
if (match) {
|
||||
const [, protocol, path] = match;
|
||||
return `${protocol}${normalizeString(path)}`;
|
||||
} else {
|
||||
return rootName;
|
||||
}
|
||||
}
|
||||
|
||||
function buildBundle(rootName, data, sourceFiles, target) {
|
||||
// when outputting to AMD and a single outfile, TypeScript makes up the module
|
||||
// specifiers which are used to define the modules, and doesn't expose them
|
||||
// publicly, so we have to try to replicate
|
||||
const sources = sourceFiles.map((sf) => sf.fileName);
|
||||
const sharedPath = commonPath(sources);
|
||||
rootName = normalizeUrl(rootName)
|
||||
.replace(sharedPath, "")
|
||||
.replace(/\.\w+$/i, "");
|
||||
// If one of the modules requires support for top-level-await, TypeScript will
|
||||
// emit the execute function as an async function. When this is the case we
|
||||
// need to bubble up the TLA to the instantiation, otherwise we instantiate
|
||||
// synchronously.
|
||||
const hasTla = data.match(/execute:\sasync\sfunction\s/);
|
||||
let instantiate;
|
||||
if (rootExports && rootExports.length) {
|
||||
instantiate = hasTla
|
||||
? `const __exp = await __instantiate("${rootName}", true);\n`
|
||||
: `const __exp = __instantiate("${rootName}", false);\n`;
|
||||
for (const rootExport of rootExports) {
|
||||
if (rootExport === "default") {
|
||||
instantiate += `export default __exp["${rootExport}"];\n`;
|
||||
} else {
|
||||
instantiate +=
|
||||
`export const ${rootExport} = __exp["${rootExport}"];\n`;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
instantiate = hasTla
|
||||
? `await __instantiate("${rootName}", true);\n`
|
||||
: `__instantiate("${rootName}", false);\n`;
|
||||
}
|
||||
const es5Bundle = target === ts.ScriptTarget.ES3 ||
|
||||
target === ts.ScriptTarget.ES5 ||
|
||||
target === ts.ScriptTarget.ES2015 ||
|
||||
target === ts.ScriptTarget.ES2016;
|
||||
return `${
|
||||
es5Bundle ? SYSTEM_LOADER_ES5 : SYSTEM_LOADER
|
||||
}\n${data}\n${instantiate}`;
|
||||
}
|
||||
|
||||
function setRootExports(program, rootModule) {
|
||||
// get a reference to the type checker, this will let us find symbols from
|
||||
// the AST.
|
||||
const checker = program.getTypeChecker();
|
||||
// get a reference to the main source file for the bundle
|
||||
const mainSourceFile = program.getSourceFile(rootModule);
|
||||
assert(mainSourceFile);
|
||||
// retrieve the internal TypeScript symbol for this AST node
|
||||
const mainSymbol = checker.getSymbolAtLocation(mainSourceFile);
|
||||
if (!mainSymbol) {
|
||||
return;
|
||||
}
|
||||
rootExports = checker
|
||||
.getExportsOfModule(mainSymbol)
|
||||
// .getExportsOfModule includes type only symbols which are exported from
|
||||
// the module, so we need to try to filter those out. While not critical
|
||||
// someone looking at the bundle would think there is runtime code behind
|
||||
// that when there isn't. There appears to be no clean way of figuring that
|
||||
// out, so inspecting SymbolFlags that might be present that are type only
|
||||
.filter(
|
||||
(sym) =>
|
||||
sym.flags & ts.SymbolFlags.Class ||
|
||||
!(
|
||||
sym.flags & ts.SymbolFlags.Interface ||
|
||||
sym.flags & ts.SymbolFlags.TypeLiteral ||
|
||||
sym.flags & ts.SymbolFlags.Signature ||
|
||||
sym.flags & ts.SymbolFlags.TypeParameter ||
|
||||
sym.flags & ts.SymbolFlags.TypeAlias ||
|
||||
sym.flags & ts.SymbolFlags.Type ||
|
||||
sym.flags & ts.SymbolFlags.Namespace ||
|
||||
sym.flags & ts.SymbolFlags.InterfaceExcludes ||
|
||||
sym.flags & ts.SymbolFlags.TypeParameterExcludes ||
|
||||
sym.flags & ts.SymbolFlags.TypeAliasExcludes
|
||||
),
|
||||
)
|
||||
.map((sym) => sym.getName());
|
||||
}
|
||||
|
||||
function runtimeCompile(request) {
|
||||
const { compilerOptions, rootNames, target, sourceFileMap } = request;
|
||||
|
||||
debug(">>> runtime compile start", {
|
||||
rootNames,
|
||||
});
|
||||
|
||||
// if there are options, convert them into TypeScript compiler options,
|
||||
// and resolve any external file references
|
||||
const result = parseCompilerOptions(
|
||||
compilerOptions,
|
||||
);
|
||||
const options = result.options;
|
||||
// TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson`
|
||||
// however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts)
|
||||
options.allowNonTsExtensions = true;
|
||||
|
||||
buildLocalSourceFileCache(sourceFileMap);
|
||||
|
||||
const state = {
|
||||
rootNames,
|
||||
emitMap: {},
|
||||
};
|
||||
legacyHostState.target = target;
|
||||
legacyHostState.writeFile = createRuntimeCompileWriteFile(state);
|
||||
const program = ts.createProgram({
|
||||
rootNames,
|
||||
options,
|
||||
host,
|
||||
});
|
||||
|
||||
const diagnostics = ts
|
||||
.getPreEmitDiagnostics(program)
|
||||
.filter(({ code }) =>
|
||||
!IGNORED_DIAGNOSTICS.includes(code) &&
|
||||
!IGNORED_COMPILE_DIAGNOSTICS.includes(code)
|
||||
);
|
||||
|
||||
const emitResult = program.emit();
|
||||
assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
|
||||
|
||||
debug("<<< runtime compile finish", {
|
||||
rootNames,
|
||||
emitMap: Object.keys(state.emitMap),
|
||||
});
|
||||
|
||||
const maybeDiagnostics = diagnostics.length
|
||||
? fromTypeScriptDiagnostic(diagnostics)
|
||||
: [];
|
||||
|
||||
return {
|
||||
diagnostics: maybeDiagnostics,
|
||||
emitMap: state.emitMap,
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeBundle(request) {
|
||||
const { compilerOptions, rootNames, target, sourceFileMap } = request;
|
||||
|
||||
debug(">>> runtime bundle start", {
|
||||
rootNames,
|
||||
});
|
||||
|
||||
// if there are options, convert them into TypeScript compiler options,
|
||||
// and resolve any external file references
|
||||
const result = parseCompilerOptions(
|
||||
compilerOptions,
|
||||
);
|
||||
const options = result.options;
|
||||
// TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson`
|
||||
// however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts)
|
||||
options.allowNonTsExtensions = true;
|
||||
|
||||
buildLocalSourceFileCache(sourceFileMap);
|
||||
|
||||
const state = {
|
||||
rootNames,
|
||||
bundleOutput: undefined,
|
||||
};
|
||||
|
||||
legacyHostState.target = target;
|
||||
legacyHostState.writeFile = createBundleWriteFile(state);
|
||||
state.options = options;
|
||||
|
||||
const program = ts.createProgram({
|
||||
rootNames,
|
||||
options,
|
||||
host,
|
||||
});
|
||||
|
||||
setRootExports(program, rootNames[0]);
|
||||
const diagnostics = ts
|
||||
.getPreEmitDiagnostics(program)
|
||||
.filter(({ code }) => !IGNORED_DIAGNOSTICS.includes(code));
|
||||
|
||||
const emitResult = program.emit();
|
||||
|
||||
assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
|
||||
|
||||
debug("<<< runtime bundle finish", {
|
||||
rootNames,
|
||||
});
|
||||
|
||||
const maybeDiagnostics = diagnostics.length
|
||||
? fromTypeScriptDiagnostic(diagnostics)
|
||||
: [];
|
||||
|
||||
return {
|
||||
diagnostics: maybeDiagnostics,
|
||||
output: state.bundleOutput,
|
||||
};
|
||||
}
|
||||
|
||||
function opCompilerRespond(msg) {
|
||||
core.jsonOpSync("op_compiler_respond", msg);
|
||||
}
|
||||
|
||||
function tsCompilerOnMessage(msg) {
|
||||
const request = msg.data;
|
||||
switch (request.type) {
|
||||
case CompilerRequestType.RuntimeCompile: {
|
||||
const result = runtimeCompile(request);
|
||||
opCompilerRespond(result);
|
||||
break;
|
||||
}
|
||||
case CompilerRequestType.RuntimeBundle: {
|
||||
const result = runtimeBundle(request);
|
||||
opCompilerRespond(result);
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
`!!! unhandled CompilerRequestType: ${request.type} (${
|
||||
CompilerRequestType[request.type]
|
||||
})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {object} Request
|
||||
* @property {Record<string, any>} config
|
||||
|
|
@ -1094,6 +678,4 @@ delete Object.prototype.__proto__;
|
|||
|
||||
globalThis.startup = startup;
|
||||
globalThis.exec = exec;
|
||||
// TODO(@kitsonk) remove when converted from legacy tsc
|
||||
globalThis.tsCompilerOnMessage = tsCompilerOnMessage;
|
||||
})(this);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue