mirror of
https://github.com/denoland/deno.git
synced 2025-07-24 05:35:33 +00:00
refactor(cli): move runTests() and runBenchmarks() to rust (#18563)
Stores the test/bench functions in rust op state during registration. The functions are wrapped in JS first so that they return a directly convertible `TestResult`/`BenchResult`. Test steps are still mostly handled in JS since they are pretty much invoked by the user. Allows removing a bunch of infrastructure for communicating between JS and rust. Allows using rust utilities for things like shuffling tests (`Vec::shuffle`). We can progressively move op and resource sanitization to rust as well. Fixes #17122. Fixes #17312.
This commit is contained in:
Nayeem Rahman
GitHub
parent
4e53bc5a94
commit
6e8618ae0f
10 changed files with 450 additions and 705 deletions
|
|
@ -2,21 +2,16 @@
|
|||
|
||||
const core = globalThis.Deno.core;
|
||||
const ops = core.ops;
|
||||
const internals = globalThis.__bootstrap.internals;
|
||||
import { setExitHandler } from "ext:runtime/30_os.js";
|
||||
import { Console } from "ext:deno_console/02_console.js";
|
||||
import { serializePermissions } from "ext:runtime/10_permissions.js";
|
||||
import { assert } from "ext:deno_web/00_infra.js";
|
||||
const primordials = globalThis.__bootstrap.primordials;
|
||||
const {
|
||||
ArrayFrom,
|
||||
ArrayPrototypeFilter,
|
||||
ArrayPrototypeJoin,
|
||||
ArrayPrototypeMap,
|
||||
ArrayPrototypePush,
|
||||
ArrayPrototypeShift,
|
||||
ArrayPrototypeSort,
|
||||
BigInt,
|
||||
DateNow,
|
||||
Error,
|
||||
FunctionPrototype,
|
||||
|
|
@ -36,6 +31,7 @@ const {
|
|||
} = primordials;
|
||||
|
||||
const opSanitizerDelayResolveQueue = [];
|
||||
let hasSetOpSanitizerDelayMacrotask = false;
|
||||
|
||||
// Even if every resource is closed by the end of a test, there can be a delay
|
||||
// until the pending ops have all finished. This function returns a promise
|
||||
|
|
@ -47,6 +43,10 @@ const opSanitizerDelayResolveQueue = [];
|
|||
// before that, though, in order to give time for worker message ops to finish
|
||||
// (since timeouts of 0 don't queue tasks in the timer queue immediately).
|
||||
function opSanitizerDelay() {
|
||||
if (!hasSetOpSanitizerDelayMacrotask) {
|
||||
core.setMacrotaskCallback(handleOpSanitizerDelayMacrotask);
|
||||
hasSetOpSanitizerDelayMacrotask = true;
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
setTimeout(() => {
|
||||
ArrayPrototypePush(opSanitizerDelayResolveQueue, resolve);
|
||||
|
|
@ -415,9 +415,28 @@ function assertExit(fn, isTest) {
|
|||
};
|
||||
}
|
||||
|
||||
function assertTestStepScopes(fn) {
|
||||
function wrapOuter(fn, desc) {
|
||||
return async function outerWrapped() {
|
||||
try {
|
||||
if (desc.ignore) {
|
||||
return "ignored";
|
||||
}
|
||||
return await fn(desc) ?? "ok";
|
||||
} catch (error) {
|
||||
return { failed: { jsError: core.destructureError(error) } };
|
||||
} finally {
|
||||
const state = MapPrototypeGet(testStates, desc.id);
|
||||
for (const childDesc of state.children) {
|
||||
stepReportResult(childDesc, { failed: "incomplete" }, 0);
|
||||
}
|
||||
state.completed = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function wrapInner(fn) {
|
||||
/** @param desc {TestDescription | TestStepDescription} */
|
||||
return async function testStepSanitizer(desc) {
|
||||
return async function innerWrapped(desc) {
|
||||
function getRunningStepDescs() {
|
||||
const results = [];
|
||||
let childDesc = desc;
|
||||
|
|
@ -458,11 +477,17 @@ function assertTestStepScopes(fn) {
|
|||
};
|
||||
}
|
||||
await fn(MapPrototypeGet(testStates, desc.id).context);
|
||||
let failedSteps = 0;
|
||||
for (const childDesc of MapPrototypeGet(testStates, desc.id).children) {
|
||||
if (!MapPrototypeGet(testStates, childDesc.id).completed) {
|
||||
const state = MapPrototypeGet(testStates, childDesc.id);
|
||||
if (!state.completed) {
|
||||
return { failed: "incompleteSteps" };
|
||||
}
|
||||
if (state.failed) {
|
||||
failedSteps++;
|
||||
}
|
||||
}
|
||||
return failedSteps == 0 ? null : { failed: { failedSteps } };
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -495,7 +520,6 @@ function withPermissions(fn, permissions) {
|
|||
* fn: TestFunction
|
||||
* origin: string,
|
||||
* location: TestLocation,
|
||||
* filteredOut: boolean,
|
||||
* ignore: boolean,
|
||||
* only: boolean.
|
||||
* sanitizeOps: boolean,
|
||||
|
|
@ -538,7 +562,6 @@ function withPermissions(fn, permissions) {
|
|||
* name: string,
|
||||
* fn: BenchFunction
|
||||
* origin: string,
|
||||
* filteredOut: boolean,
|
||||
* ignore: boolean,
|
||||
* only: boolean.
|
||||
* sanitizeExit: boolean,
|
||||
|
|
@ -546,14 +569,8 @@ function withPermissions(fn, permissions) {
|
|||
* }} BenchDescription
|
||||
*/
|
||||
|
||||
/** @type {TestDescription[]} */
|
||||
const testDescs = [];
|
||||
/** @type {Map<number, TestState | TestStepState>} */
|
||||
const testStates = new Map();
|
||||
/** @type {BenchDescription[]} */
|
||||
const benchDescs = [];
|
||||
let isTestSubcommand = false;
|
||||
let isBenchSubcommand = false;
|
||||
|
||||
// Main test function provided by Deno.
|
||||
function test(
|
||||
|
|
@ -561,7 +578,7 @@ function test(
|
|||
optionsOrFn,
|
||||
maybeFn,
|
||||
) {
|
||||
if (!isTestSubcommand) {
|
||||
if (typeof ops.op_register_test != "function") {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -647,19 +664,17 @@ function test(
|
|||
|
||||
// Delete this prop in case the user passed it. It's used to detect steps.
|
||||
delete testDesc.parent;
|
||||
testDesc.origin = getTestOrigin();
|
||||
const jsError = core.destructureError(new Error());
|
||||
testDesc.location = {
|
||||
fileName: jsError.frames[1].fileName,
|
||||
lineNumber: jsError.frames[1].lineNumber,
|
||||
columnNumber: jsError.frames[1].columnNumber,
|
||||
};
|
||||
testDesc.fn = wrapTest(testDesc);
|
||||
|
||||
const { id, filteredOut } = ops.op_register_test(testDesc);
|
||||
const { id, origin } = ops.op_register_test(testDesc);
|
||||
testDesc.id = id;
|
||||
testDesc.filteredOut = filteredOut;
|
||||
|
||||
ArrayPrototypePush(testDescs, testDesc);
|
||||
testDesc.origin = origin;
|
||||
MapPrototypeSet(testStates, testDesc.id, {
|
||||
context: createTestContext(testDesc),
|
||||
children: [],
|
||||
|
|
@ -673,7 +688,7 @@ function bench(
|
|||
optionsOrFn,
|
||||
maybeFn,
|
||||
) {
|
||||
if (!isBenchSubcommand) {
|
||||
if (typeof ops.op_register_bench != "function") {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -756,36 +771,13 @@ function bench(
|
|||
benchDesc = { ...defaults, ...nameOrFnOrOptions, fn, name };
|
||||
}
|
||||
|
||||
benchDesc.origin = getBenchOrigin();
|
||||
const AsyncFunction = (async () => {}).constructor;
|
||||
benchDesc.async = AsyncFunction === benchDesc.fn.constructor;
|
||||
benchDesc.fn = wrapBenchmark(benchDesc);
|
||||
|
||||
const { id, filteredOut } = ops.op_register_bench(benchDesc);
|
||||
const { id, origin } = ops.op_register_bench(benchDesc);
|
||||
benchDesc.id = id;
|
||||
benchDesc.filteredOut = filteredOut;
|
||||
|
||||
ArrayPrototypePush(benchDescs, benchDesc);
|
||||
}
|
||||
|
||||
async function runTest(desc) {
|
||||
if (desc.ignore) {
|
||||
return "ignored";
|
||||
}
|
||||
let testFn = wrapTestFnWithSanitizers(desc.fn, desc);
|
||||
if (!("parent" in desc) && desc.permissions) {
|
||||
testFn = withPermissions(
|
||||
testFn,
|
||||
desc.permissions,
|
||||
);
|
||||
}
|
||||
try {
|
||||
const result = await testFn(desc);
|
||||
if (result) return result;
|
||||
const failedSteps = failedChildStepsCount(desc);
|
||||
return failedSteps === 0 ? "ok" : { failed: { failedSteps } };
|
||||
} catch (error) {
|
||||
return { failed: { jsError: core.destructureError(error) } };
|
||||
}
|
||||
benchDesc.origin = origin;
|
||||
}
|
||||
|
||||
function compareMeasurements(a, b) {
|
||||
|
|
@ -808,8 +800,7 @@ function benchStats(n, highPrecision, avg, min, max, all) {
|
|||
};
|
||||
}
|
||||
|
||||
async function benchMeasure(timeBudget, desc) {
|
||||
const fn = desc.fn;
|
||||
async function benchMeasure(timeBudget, fn, async) {
|
||||
let n = 0;
|
||||
let avg = 0;
|
||||
let wavg = 0;
|
||||
|
|
@ -823,7 +814,7 @@ async function benchMeasure(timeBudget, desc) {
|
|||
let iterations = 20;
|
||||
let budget = 10 * 1e6;
|
||||
|
||||
if (!desc.async) {
|
||||
if (!async) {
|
||||
while (budget > 0 || iterations-- > 0) {
|
||||
const t1 = benchNow();
|
||||
|
||||
|
|
@ -854,7 +845,7 @@ async function benchMeasure(timeBudget, desc) {
|
|||
let iterations = 10;
|
||||
let budget = timeBudget * 1e6;
|
||||
|
||||
if (!desc.async) {
|
||||
if (!async) {
|
||||
while (budget > 0 || iterations-- > 0) {
|
||||
const t1 = benchNow();
|
||||
|
||||
|
|
@ -887,7 +878,7 @@ async function benchMeasure(timeBudget, desc) {
|
|||
let iterations = 10;
|
||||
let budget = timeBudget * 1e6;
|
||||
|
||||
if (!desc.async) {
|
||||
if (!async) {
|
||||
while (budget > 0 || iterations-- > 0) {
|
||||
const t1 = benchNow();
|
||||
for (let c = 0; c < lowPrecisionThresholdInNs; c++) fn();
|
||||
|
|
@ -920,173 +911,49 @@ async function benchMeasure(timeBudget, desc) {
|
|||
return benchStats(n, wavg > lowPrecisionThresholdInNs, avg, min, max, all);
|
||||
}
|
||||
|
||||
async function runBench(desc) {
|
||||
let token = null;
|
||||
/** Wrap a user benchmark function in one which returns a structured result. */
|
||||
function wrapBenchmark(desc) {
|
||||
const fn = desc.fn;
|
||||
return async function outerWrapped() {
|
||||
let token = null;
|
||||
const originalConsole = globalThis.console;
|
||||
|
||||
try {
|
||||
if (desc.permissions) {
|
||||
token = pledgePermissions(desc.permissions);
|
||||
}
|
||||
|
||||
if (desc.sanitizeExit) {
|
||||
setExitHandler((exitCode) => {
|
||||
assert(
|
||||
false,
|
||||
`Bench attempted to exit with exit code: ${exitCode}`,
|
||||
);
|
||||
try {
|
||||
globalThis.console = new Console((s) => {
|
||||
ops.op_dispatch_bench_event({ output: s });
|
||||
});
|
||||
|
||||
if (desc.permissions) {
|
||||
token = pledgePermissions(desc.permissions);
|
||||
}
|
||||
|
||||
if (desc.sanitizeExit) {
|
||||
setExitHandler((exitCode) => {
|
||||
assert(
|
||||
false,
|
||||
`Bench attempted to exit with exit code: ${exitCode}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
const benchTimeInMs = 500;
|
||||
const stats = await benchMeasure(benchTimeInMs, fn, desc.async);
|
||||
|
||||
return { ok: stats };
|
||||
} catch (error) {
|
||||
return { failed: core.destructureError(error) };
|
||||
} finally {
|
||||
globalThis.console = originalConsole;
|
||||
if (bench.sanitizeExit) setExitHandler(null);
|
||||
if (token !== null) restorePermissions(token);
|
||||
}
|
||||
|
||||
const benchTimeInMs = 500;
|
||||
const stats = await benchMeasure(benchTimeInMs, desc);
|
||||
|
||||
return { ok: stats };
|
||||
} catch (error) {
|
||||
return { failed: core.destructureError(error) };
|
||||
} finally {
|
||||
if (bench.sanitizeExit) setExitHandler(null);
|
||||
if (token !== null) restorePermissions(token);
|
||||
}
|
||||
}
|
||||
|
||||
let origin = null;
|
||||
|
||||
function getTestOrigin() {
|
||||
if (origin == null) {
|
||||
origin = ops.op_get_test_origin();
|
||||
}
|
||||
return origin;
|
||||
}
|
||||
|
||||
function getBenchOrigin() {
|
||||
if (origin == null) {
|
||||
origin = ops.op_get_bench_origin();
|
||||
}
|
||||
return origin;
|
||||
};
|
||||
}
|
||||
|
||||
function benchNow() {
|
||||
return ops.op_bench_now();
|
||||
}
|
||||
|
||||
function enableTest() {
|
||||
isTestSubcommand = true;
|
||||
}
|
||||
|
||||
function enableBench() {
|
||||
isBenchSubcommand = true;
|
||||
}
|
||||
|
||||
async function runTests({
|
||||
shuffle = null,
|
||||
} = {}) {
|
||||
core.setMacrotaskCallback(handleOpSanitizerDelayMacrotask);
|
||||
|
||||
const origin = getTestOrigin();
|
||||
const only = ArrayPrototypeFilter(testDescs, (test) => test.only);
|
||||
const filtered = ArrayPrototypeFilter(
|
||||
only.length > 0 ? only : testDescs,
|
||||
(desc) => !desc.filteredOut,
|
||||
);
|
||||
|
||||
ops.op_dispatch_test_event({
|
||||
plan: {
|
||||
origin,
|
||||
total: filtered.length,
|
||||
filteredOut: testDescs.length - filtered.length,
|
||||
usedOnly: only.length > 0,
|
||||
},
|
||||
});
|
||||
|
||||
if (shuffle !== null) {
|
||||
// http://en.wikipedia.org/wiki/Linear_congruential_generator
|
||||
// Use BigInt for everything because the random seed is u64.
|
||||
const nextInt = function (state) {
|
||||
const m = 0x80000000n;
|
||||
const a = 1103515245n;
|
||||
const c = 12345n;
|
||||
|
||||
return function (max) {
|
||||
return state = ((a * state + c) % m) % BigInt(max);
|
||||
};
|
||||
}(BigInt(shuffle));
|
||||
|
||||
for (let i = filtered.length - 1; i > 0; i--) {
|
||||
const j = nextInt(i);
|
||||
[filtered[i], filtered[j]] = [filtered[j], filtered[i]];
|
||||
}
|
||||
}
|
||||
|
||||
for (const desc of filtered) {
|
||||
if (ops.op_tests_should_stop()) {
|
||||
break;
|
||||
}
|
||||
ops.op_dispatch_test_event({ wait: desc.id });
|
||||
const earlier = DateNow();
|
||||
const result = await runTest(desc);
|
||||
const elapsed = DateNow() - earlier;
|
||||
const state = MapPrototypeGet(testStates, desc.id);
|
||||
state.completed = true;
|
||||
for (const childDesc of state.children) {
|
||||
stepReportResult(childDesc, { failed: "incomplete" }, 0);
|
||||
}
|
||||
ops.op_dispatch_test_event({
|
||||
result: [desc.id, result, elapsed],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function runBenchmarks() {
|
||||
core.setMacrotaskCallback(handleOpSanitizerDelayMacrotask);
|
||||
|
||||
const origin = getBenchOrigin();
|
||||
const originalConsole = globalThis.console;
|
||||
|
||||
globalThis.console = new Console((s) => {
|
||||
ops.op_dispatch_bench_event({ output: s });
|
||||
});
|
||||
|
||||
const only = ArrayPrototypeFilter(benchDescs, (bench) => bench.only);
|
||||
const filtered = ArrayPrototypeFilter(
|
||||
only.length > 0 ? only : benchDescs,
|
||||
(desc) => !desc.filteredOut && !desc.ignore,
|
||||
);
|
||||
|
||||
let groups = new Set();
|
||||
// make sure ungrouped benchmarks are placed above grouped
|
||||
groups.add(undefined);
|
||||
|
||||
for (const desc of filtered) {
|
||||
desc.group ||= undefined;
|
||||
groups.add(desc.group);
|
||||
}
|
||||
|
||||
groups = ArrayFrom(groups);
|
||||
ArrayPrototypeSort(
|
||||
filtered,
|
||||
(a, b) => groups.indexOf(a.group) - groups.indexOf(b.group),
|
||||
);
|
||||
|
||||
ops.op_dispatch_bench_event({
|
||||
plan: {
|
||||
origin,
|
||||
total: filtered.length,
|
||||
usedOnly: only.length > 0,
|
||||
names: ArrayPrototypeMap(filtered, (desc) => desc.name),
|
||||
},
|
||||
});
|
||||
|
||||
for (const desc of filtered) {
|
||||
desc.baseline = !!desc.baseline;
|
||||
ops.op_dispatch_bench_event({ wait: desc.id });
|
||||
ops.op_dispatch_bench_event({
|
||||
result: [desc.id, await runBench(desc)],
|
||||
});
|
||||
}
|
||||
|
||||
globalThis.console = originalConsole;
|
||||
}
|
||||
|
||||
function getFullName(desc) {
|
||||
if ("parent" in desc) {
|
||||
return `${getFullName(desc.parent)} ... ${desc.name}`;
|
||||
|
|
@ -1108,13 +975,6 @@ function stepReportResult(desc, result, elapsed) {
|
|||
});
|
||||
}
|
||||
|
||||
function failedChildStepsCount(desc) {
|
||||
return ArrayPrototypeFilter(
|
||||
MapPrototypeGet(testStates, desc.id).children,
|
||||
(d) => MapPrototypeGet(testStates, d.id).failed,
|
||||
).length;
|
||||
}
|
||||
|
||||
/** @param desc {TestDescription | TestStepDescription} */
|
||||
function createTestContext(desc) {
|
||||
let parent;
|
||||
|
|
@ -1191,7 +1051,6 @@ function createTestContext(desc) {
|
|||
stepDesc.sanitizeOps ??= desc.sanitizeOps;
|
||||
stepDesc.sanitizeResources ??= desc.sanitizeResources;
|
||||
stepDesc.sanitizeExit ??= desc.sanitizeExit;
|
||||
stepDesc.origin = getTestOrigin();
|
||||
const jsError = core.destructureError(new Error());
|
||||
stepDesc.location = {
|
||||
fileName: jsError.frames[1].fileName,
|
||||
|
|
@ -1202,8 +1061,10 @@ function createTestContext(desc) {
|
|||
stepDesc.parent = desc;
|
||||
stepDesc.rootId = rootId;
|
||||
stepDesc.rootName = rootName;
|
||||
const { id } = ops.op_register_test_step(stepDesc);
|
||||
stepDesc.fn = wrapTest(stepDesc);
|
||||
const { id, origin } = ops.op_register_test_step(stepDesc);
|
||||
stepDesc.id = id;
|
||||
stepDesc.origin = origin;
|
||||
const state = {
|
||||
context: createTestContext(stepDesc),
|
||||
children: [],
|
||||
|
|
@ -1218,10 +1079,9 @@ function createTestContext(desc) {
|
|||
|
||||
ops.op_dispatch_test_event({ stepWait: stepDesc.id });
|
||||
const earlier = DateNow();
|
||||
const result = await runTest(stepDesc);
|
||||
const result = await stepDesc.fn(stepDesc);
|
||||
const elapsed = DateNow() - earlier;
|
||||
state.failed = !!result.failed;
|
||||
state.completed = true;
|
||||
stepReportResult(stepDesc, result, elapsed);
|
||||
return result == "ok";
|
||||
},
|
||||
|
|
@ -1229,37 +1089,29 @@ function createTestContext(desc) {
|
|||
}
|
||||
|
||||
/**
|
||||
* Wrap a user test function in one which returns a structured result.
|
||||
* @template T {Function}
|
||||
* @param testFn {T}
|
||||
* @param opts {{
|
||||
* sanitizeOps: boolean,
|
||||
* sanitizeResources: boolean,
|
||||
* sanitizeExit: boolean,
|
||||
* }}
|
||||
* @param desc {TestDescription | TestStepDescription}
|
||||
* @returns {T}
|
||||
*/
|
||||
function wrapTestFnWithSanitizers(testFn, opts) {
|
||||
testFn = assertTestStepScopes(testFn);
|
||||
|
||||
if (opts.sanitizeOps) {
|
||||
function wrapTest(desc) {
|
||||
let testFn = wrapInner(desc.fn);
|
||||
if (desc.sanitizeOps) {
|
||||
testFn = assertOps(testFn);
|
||||
}
|
||||
if (opts.sanitizeResources) {
|
||||
if (desc.sanitizeResources) {
|
||||
testFn = assertResources(testFn);
|
||||
}
|
||||
if (opts.sanitizeExit) {
|
||||
if (desc.sanitizeExit) {
|
||||
testFn = assertExit(testFn, true);
|
||||
}
|
||||
return testFn;
|
||||
if (!("parent" in desc) && desc.permissions) {
|
||||
testFn = withPermissions(testFn, desc.permissions);
|
||||
}
|
||||
return wrapOuter(testFn, desc);
|
||||
}
|
||||
|
||||
internals.testing = {
|
||||
runTests,
|
||||
runBenchmarks,
|
||||
enableTest,
|
||||
enableBench,
|
||||
};
|
||||
|
||||
import { denoNs } from "ext:runtime/90_deno_ns.js";
|
||||
denoNs.bench = bench;
|
||||
denoNs.test = test;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue