diff --git a/benching/mod.ts b/benching/mod.ts deleted file mode 100644 index 6fa442d032..0000000000 --- a/benching/mod.ts +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. - -const { exit, noColor } = Deno; - -interface BenchmarkClock { - start: number; - stop: number; -} - -/** Provides methods for starting and stopping a benchmark clock. */ -export interface BenchmarkTimer { - start: () => void; - stop: () => void; -} - -/** Defines a benchmark through a named function. */ -export interface BenchmarkFunction { - (b: BenchmarkTimer): void | Promise; - name: string; -} - -/** Defines a benchmark definition with configurable runs. */ -export interface BenchmarkDefinition { - func: BenchmarkFunction; - name: string; - runs?: number; -} - -/** Defines runBenchmark's run constraints by matching benchmark names. */ -export interface BenchmarkRunOptions { - only?: RegExp; - skip?: RegExp; -} - -function red(text: string): string { - return noColor ? text : `\x1b[31m${text}\x1b[0m`; -} - -function blue(text: string): string { - return noColor ? text : `\x1b[34m${text}\x1b[0m`; -} - -function verifyOr1Run(runs?: number): number { - return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1; -} - -function assertTiming(clock: BenchmarkClock): void { - // NaN indicates that a benchmark has not been timed properly - if (!clock.stop) { - throw new Error("The benchmark timer's stop method must be called"); - } else if (!clock.start) { - throw new Error("The benchmark timer's start method must be called"); - } else if (clock.start > clock.stop) { - throw new Error( - "The benchmark timer's start method must be called before its " + - "stop method" - ); - } -} - -function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer { - return { - start(): void { - clock.start = Date.now(); - }, - stop(): void { - clock.stop = Date.now(); - } - }; -} - -const candidates: BenchmarkDefinition[] = []; - -/** Registers a benchmark as a candidate for the runBenchmarks executor. */ -export function bench( - benchmark: BenchmarkDefinition | BenchmarkFunction -): void { - if (!benchmark.name) { - throw new Error("The benchmark function must not be anonymous"); - } - if (typeof benchmark === "function") { - candidates.push({ name: benchmark.name, runs: 1, func: benchmark }); - } else { - candidates.push({ - name: benchmark.name, - runs: verifyOr1Run(benchmark.runs), - func: benchmark.func - }); - } -} - -/** Runs all registered and non-skipped benchmarks serially. */ -export async function runBenchmarks({ - only = /[^\s]/, - skip = /^\s*$/ -}: BenchmarkRunOptions = {}): Promise { - // Filtering candidates by the "only" and "skip" constraint - const benchmarks: BenchmarkDefinition[] = candidates.filter( - ({ name }) => only.test(name) && !skip.test(name) - ); - // Init main counters and error flag - const filtered = candidates.length - benchmarks.length; - let measured = 0; - let failed = false; - // Setting up a shared benchmark clock and timer - const clock: BenchmarkClock = { start: NaN, stop: NaN }; - const b = createBenchmarkTimer(clock); - // Iterating given benchmark definitions (await-in-loop) - console.log( - "running", - benchmarks.length, - `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` - ); - for (const { name, runs = 0, func } of benchmarks) { - // See https://github.com/denoland/deno/pull/1452 about groupCollapsed - console.groupCollapsed(`benchmark ${name} ... `); - // Trying benchmark.func - let result = ""; - try { - if (runs === 1) { - // b is a benchmark timer interfacing an unset (NaN) benchmark clock - await func(b); - // Making sure the benchmark was started/stopped properly - assertTiming(clock); - result = `${clock.stop - clock.start}ms`; - } else if (runs > 1) { - // Averaging runs - let pendingRuns = runs; - let totalMs = 0; - // Would be better 2 not run these serially - while (true) { - // b is a benchmark timer interfacing an unset (NaN) benchmark clock - await func(b); - // Making sure the benchmark was started/stopped properly - assertTiming(clock); - // Summing up - totalMs += clock.stop - clock.start; - // Resetting the benchmark clock - clock.start = clock.stop = NaN; - // Once all ran - if (!--pendingRuns) { - result = `${runs} runs avg: ${totalMs / runs}ms`; - break; - } - } - } - } catch (err) { - failed = true; - console.groupEnd(); - console.error(red(err.stack)); - break; - } - // Reporting - console.log(blue(result)); - console.groupEnd(); - measured++; - // Resetting the benchmark clock - clock.start = clock.stop = NaN; - } - // Closing results - console.log( - `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` + - `${measured} measured; ${filtered} filtered` - ); - // Making sure the program exit code is not zero in case of failure - if (failed) { - setTimeout(() => exit(1), 0); - } -} diff --git a/benching/readme.md b/benching/readme.md deleted file mode 100644 index 937c9fcc4c..0000000000 --- a/benching/readme.md +++ /dev/null @@ -1,86 +0,0 @@ -# benching - -Basic benchmarking module. Provides flintstone millisecond resolution. - -## Import - -```ts -import * as benching from "https://deno.land/std/benching/mod.ts"; -``` - -## Usage - -```ts -import { - BenchmarkTimer, - runBenchmarks, - bench -} from "https://deno.land/std/benching/mod.ts"; - -// Simple -bench(function forIncrementX1e9(b: BenchmarkTimer) { - b.start(); - for (let i = 0; i < 1e9; i++); - b.stop(); -}); - -// Reporting average measured time for $runs runs of func -bench({ - name: "runs100ForIncrementX1e6", - runs: 100, - func(b: BenchmarkTimer) { - b.start(); - for (let i: number = 0; i < 1e6; i++); - b.stop(); - } -}); - -// Itsabug -bench(function throwing(b) { - b.start(); - // Throws bc the timer's stop method is never called -}); - -// Bench control -runBenchmarks({ skip: /throw/ }); -``` - -## API - -#### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void` - -Registers a benchmark that will be run once `runBenchmarks` is called. - -#### `runBenchmarks(opts?: BenchmarkRunOptions): Promise` - -Runs all registered benchmarks serially. Filtering can be applied by setting -`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names. - -#### Other exports - -```ts -/** Provides methods for starting and stopping a benchmark clock. */ -export interface BenchmarkTimer { - start: () => void; - stop: () => void; -} - -/** Defines a benchmark through a named function. */ -export type BenchmarkFunction = { - (b: BenchmarkTimer): void | Promise; - name: string; -}; - -/** Defines a benchmark definition with configurable runs. */ -export interface BenchmarkDefinition { - func: BenchmarkFunction; - name: string; - runs?: number; -} - -/** Defines runBenchmark's run constraints by matching benchmark names. */ -export interface BenchmarkRunOptions { - only?: RegExp; - skip?: RegExp; -} -``` diff --git a/test.ts b/test.ts index 4c51943d3e..400cf323b9 100755 --- a/test.ts +++ b/test.ts @@ -1,6 +1,5 @@ #!/usr/bin/env deno -A // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -import "./benching/test.ts"; import "./colors/test.ts"; import "./datetime/test.ts"; import "./examples/test.ts"; diff --git a/testing/README.md b/testing/README.md index c134dd9129..05cb8b92e4 100644 --- a/testing/README.md +++ b/testing/README.md @@ -131,3 +131,77 @@ test(async function fails() { }); }); ``` + +### Benching Usage + +Basic usage: + +```ts +import { runBenchmarks, bench } from "https://deno.land/std/testing/bench.ts"; + +bench(function forIncrementX1e9(b) { + b.start(); + for (let i = 0; i < 1e9; i++); + b.stop(); +}); + +runBenchmarks(); +``` + +Averaging execution time over multiple runs: + +```ts +bench({ + name: "runs100ForIncrementX1e6", + runs: 100, + func(b) { + b.start(); + for (let i = 0; i < 1e6; i++); + b.stop(); + } +}); +``` + +#### Benching API + +##### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void` + +Registers a benchmark that will be run once `runBenchmarks` is called. + +##### `runBenchmarks(opts?: BenchmarkRunOptions): Promise` + +Runs all registered benchmarks serially. Filtering can be applied by setting +`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names. + +##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise` + +Runs specified benchmarks if the enclosing script is main. + +##### Other exports + +```ts +/** Provides methods for starting and stopping a benchmark clock. */ +export interface BenchmarkTimer { + start: () => void; + stop: () => void; +} + +/** Defines a benchmark through a named function. */ +export interface BenchmarkFunction { + (b: BenchmarkTimer): void | Promise; + name: string; +} + +/** Defines a benchmark definition with configurable runs. */ +export interface BenchmarkDefinition { + func: BenchmarkFunction; + name: string; + runs?: number; +} + +/** Defines runBenchmark's run constraints by matching benchmark names. */ +export interface BenchmarkRunOptions { + only?: RegExp; + skip?: RegExp; +} +``` diff --git a/testing/bench.ts b/testing/bench.ts index bc2e569d2d..0094f62924 100644 --- a/testing/bench.ts +++ b/testing/bench.ts @@ -1,16 +1,179 @@ -import { bench, runBenchmarks } from "./../benching/mod.ts"; -import { runTests } from "./mod.ts"; +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -bench(async function testingSerial(b) { - b.start(); - await runTests(); - b.stop(); -}); +const { exit, noColor } = Deno; -bench(async function testingParallel(b) { - b.start(); - await runTests({ parallel: true }); - b.stop(); -}); +interface BenchmarkClock { + start: number; + stop: number; +} -runBenchmarks({ only: /testing/ }); +/** Provides methods for starting and stopping a benchmark clock. */ +export interface BenchmarkTimer { + start: () => void; + stop: () => void; +} + +/** Defines a benchmark through a named function. */ +export interface BenchmarkFunction { + (b: BenchmarkTimer): void | Promise; + name: string; +} + +/** Defines a benchmark definition with configurable runs. */ +export interface BenchmarkDefinition { + func: BenchmarkFunction; + name: string; + runs?: number; +} + +/** Defines runBenchmark's run constraints by matching benchmark names. */ +export interface BenchmarkRunOptions { + only?: RegExp; + skip?: RegExp; +} + +function red(text: string): string { + return noColor ? text : `\x1b[31m${text}\x1b[0m`; +} + +function blue(text: string): string { + return noColor ? text : `\x1b[34m${text}\x1b[0m`; +} + +function verifyOr1Run(runs?: number): number { + return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1; +} + +function assertTiming(clock: BenchmarkClock): void { + // NaN indicates that a benchmark has not been timed properly + if (!clock.stop) { + throw new Error("The benchmark timer's stop method must be called"); + } else if (!clock.start) { + throw new Error("The benchmark timer's start method must be called"); + } else if (clock.start > clock.stop) { + throw new Error( + "The benchmark timer's start method must be called before its " + + "stop method" + ); + } +} + +function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer { + return { + start(): void { + clock.start = Date.now(); + }, + stop(): void { + clock.stop = Date.now(); + } + }; +} + +const candidates: BenchmarkDefinition[] = []; + +/** Registers a benchmark as a candidate for the runBenchmarks executor. */ +export function bench( + benchmark: BenchmarkDefinition | BenchmarkFunction +): void { + if (!benchmark.name) { + throw new Error("The benchmark function must not be anonymous"); + } + if (typeof benchmark === "function") { + candidates.push({ name: benchmark.name, runs: 1, func: benchmark }); + } else { + candidates.push({ + name: benchmark.name, + runs: verifyOr1Run(benchmark.runs), + func: benchmark.func + }); + } +} + +/** Runs all registered and non-skipped benchmarks serially. */ +export async function runBenchmarks({ + only = /[^\s]/, + skip = /^\s*$/ +}: BenchmarkRunOptions = {}): Promise { + // Filtering candidates by the "only" and "skip" constraint + const benchmarks: BenchmarkDefinition[] = candidates.filter( + ({ name }) => only.test(name) && !skip.test(name) + ); + // Init main counters and error flag + const filtered = candidates.length - benchmarks.length; + let measured = 0; + let failed = false; + // Setting up a shared benchmark clock and timer + const clock: BenchmarkClock = { start: NaN, stop: NaN }; + const b = createBenchmarkTimer(clock); + // Iterating given benchmark definitions (await-in-loop) + console.log( + "running", + benchmarks.length, + `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` + ); + for (const { name, runs = 0, func } of benchmarks) { + // See https://github.com/denoland/deno/pull/1452 about groupCollapsed + console.groupCollapsed(`benchmark ${name} ... `); + // Trying benchmark.func + let result = ""; + try { + if (runs === 1) { + // b is a benchmark timer interfacing an unset (NaN) benchmark clock + await func(b); + // Making sure the benchmark was started/stopped properly + assertTiming(clock); + result = `${clock.stop - clock.start}ms`; + } else if (runs > 1) { + // Averaging runs + let pendingRuns = runs; + let totalMs = 0; + // Would be better 2 not run these serially + while (true) { + // b is a benchmark timer interfacing an unset (NaN) benchmark clock + await func(b); + // Making sure the benchmark was started/stopped properly + assertTiming(clock); + // Summing up + totalMs += clock.stop - clock.start; + // Resetting the benchmark clock + clock.start = clock.stop = NaN; + // Once all ran + if (!--pendingRuns) { + result = `${runs} runs avg: ${totalMs / runs}ms`; + break; + } + } + } + } catch (err) { + failed = true; + console.groupEnd(); + console.error(red(err.stack)); + break; + } + // Reporting + console.log(blue(result)); + console.groupEnd(); + measured++; + // Resetting the benchmark clock + clock.start = clock.stop = NaN; + } + // Closing results + console.log( + `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` + + `${measured} measured; ${filtered} filtered` + ); + // Making sure the program exit code is not zero in case of failure + if (failed) { + setTimeout(() => exit(1), 0); + } +} + +/** Runs specified benchmarks if the enclosing script is main. */ +export async function runIfMain( + meta: ImportMeta, + opts?: BenchmarkRunOptions +): Promise { + if (meta.main) { + return runBenchmarks(opts); + } +} diff --git a/benching/example.ts b/testing/bench_example.ts similarity index 70% rename from benching/example.ts rename to testing/bench_example.ts index 67555239d2..86e25b9a6c 100644 --- a/benching/example.ts +++ b/testing/bench_example.ts @@ -1,7 +1,7 @@ -// https://deno.land/std/benching/mod.ts -import { BenchmarkTimer, runBenchmarks, bench } from "./mod.ts"; +// https://deno.land/std/testing/bench.ts +import { BenchmarkTimer, bench, runIfMain } from "./bench.ts"; -// Simple +// Basic bench(function forIncrementX1e9(b: BenchmarkTimer) { b.start(); for (let i = 0; i < 1e9; i++); @@ -12,7 +12,7 @@ bench(function forIncrementX1e9(b: BenchmarkTimer) { bench({ name: "runs100ForIncrementX1e6", runs: 100, - func(b: BenchmarkTimer) { + func(b) { b.start(); for (let i = 0; i < 1e6; i++); b.stop(); @@ -26,4 +26,4 @@ bench(function throwing(b) { }); // Bench control -runBenchmarks({ skip: /throw/ }); +runIfMain(import.meta, { skip: /throw/ }); diff --git a/benching/test.ts b/testing/bench_test.ts similarity index 61% rename from benching/test.ts rename to testing/bench_test.ts index 34f7b00d29..345f104a66 100644 --- a/benching/test.ts +++ b/testing/bench_test.ts @@ -1,22 +1,22 @@ -import { test } from "../testing/mod.ts"; -import { bench, runBenchmarks, BenchmarkTimer } from "./mod.ts"; +import { test, runIfMain } from "./mod.ts"; +import { bench, runBenchmarks } from "./bench.ts"; -import "example.ts"; +import "./bench_example.ts"; test(async function benching() { - bench(function forIncrementX1e9(b: BenchmarkTimer) { + bench(function forIncrementX1e9(b) { b.start(); for (let i = 0; i < 1e9; i++); b.stop(); }); - bench(function forDecrementX1e9(b: BenchmarkTimer) { + bench(function forDecrementX1e9(b) { b.start(); for (let i = 1e9; i > 0; i--); b.stop(); }); - bench(async function forAwaitFetchDenolandX10(b: BenchmarkTimer) { + bench(async function forAwaitFetchDenolandX10(b) { b.start(); for (let i = 0; i < 10; i++) { await fetch("https://deno.land/"); @@ -24,7 +24,7 @@ test(async function benching() { b.stop(); }); - bench(async function promiseAllFetchDenolandX10(b: BenchmarkTimer) { + bench(async function promiseAllFetchDenolandX10(b) { const urls = new Array(10).fill("https://deno.land/"); b.start(); await Promise.all(urls.map((denoland: string) => fetch(denoland))); @@ -34,17 +34,19 @@ test(async function benching() { bench({ name: "runs100ForIncrementX1e6", runs: 100, - func(b: BenchmarkTimer) { + func(b) { b.start(); for (let i = 0; i < 1e6; i++); b.stop(); } }); - bench(function throwing(b: BenchmarkTimer) { + bench(function throwing(b) { b.start(); // Throws bc the timer's stop method is never called }); await runBenchmarks({ skip: /throw/ }); }); + +runIfMain(import.meta); diff --git a/testing/test.ts b/testing/test.ts index 367e280228..2359cba2ea 100644 --- a/testing/test.ts +++ b/testing/test.ts @@ -11,6 +11,7 @@ import "./format_test.ts"; import "./diff_test.ts"; import "./pretty_test.ts"; import "./asserts_test.ts"; +import "./bench_test.ts"; test(function testingAssertEqualActualUncoercable() { let didThrow = false; @@ -251,4 +252,4 @@ test(async function testingThrowsAsyncMsgNotIncludes() { assert(didThrow); }); -runIfMain(import.meta, { parallel: true }); +runIfMain(import.meta); diff --git a/testing/testing_bench.ts b/testing/testing_bench.ts new file mode 100644 index 0000000000..0cc2f233b0 --- /dev/null +++ b/testing/testing_bench.ts @@ -0,0 +1,18 @@ +import { bench, runIfMain } from "./bench.ts"; +import { runTests } from "./mod.ts"; + +import "./asserts_test.ts"; + +bench(async function testingSerial(b) { + b.start(); + await runTests(); + b.stop(); +}); + +bench(async function testingParallel(b) { + b.start(); + await runTests({ parallel: true }); + b.stop(); +}); + +runIfMain(import.meta);