foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-26 16:09:27 -05:00
Code Issues Packages 1 Wiki Activity

Move benching into testing. (denoland/deno_std#258)

Browse source 
Original: 4de86f04de
This commit is contained in:
chiefbiiko 2019-03-11 19:21:13 +01:00 committed by Ryan Dahl
parent d4ba2978a6
commit 4a97a6e67e
9 changed files with 286 additions and 284 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

169
benching/mod.ts
View file

@ -1,169 +0,0 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
const { exit, noColor } = Deno;
interface BenchmarkClock {
start: number;
stop: number;
}
/** Provides methods for starting and stopping a benchmark clock. */
export interface BenchmarkTimer {
start: () => void;
stop: () => void;
}
/** Defines a benchmark through a named function. */
export interface BenchmarkFunction {
(b: BenchmarkTimer): void | Promise<void>;
name: string;
}
/** Defines a benchmark definition with configurable runs. */
export interface BenchmarkDefinition {
func: BenchmarkFunction;
name: string;
runs?: number;
}
/** Defines runBenchmark's run constraints by matching benchmark names. */
export interface BenchmarkRunOptions {
only?: RegExp;
skip?: RegExp;
}
function red(text: string): string {
return noColor ? text : `\x1b[31m${text}\x1b[0m`;
}
function blue(text: string): string {
return noColor ? text : `\x1b[34m${text}\x1b[0m`;
}
function verifyOr1Run(runs?: number): number {
return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
}
function assertTiming(clock: BenchmarkClock): void {
// NaN indicates that a benchmark has not been timed properly
if (!clock.stop) {
throw new Error("The benchmark timer's stop method must be called");
} else if (!clock.start) {
throw new Error("The benchmark timer's start method must be called");
} else if (clock.start > clock.stop) {
throw new Error(
"The benchmark timer's start method must be called before its " +
"stop method"
);
}
}
function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer {
return {
start(): void {
clock.start = Date.now();
},
stop(): void {
clock.stop = Date.now();
}
};
}
const candidates: BenchmarkDefinition[] = [];
/** Registers a benchmark as a candidate for the runBenchmarks executor. */
export function bench(
benchmark: BenchmarkDefinition | BenchmarkFunction
): void {
if (!benchmark.name) {
throw new Error("The benchmark function must not be anonymous");
}
if (typeof benchmark === "function") {
candidates.push({ name: benchmark.name, runs: 1, func: benchmark });
} else {
candidates.push({
name: benchmark.name,
runs: verifyOr1Run(benchmark.runs),
func: benchmark.func
});
}
}
/** Runs all registered and non-skipped benchmarks serially. */
export async function runBenchmarks({
only = /[^\s]/,
skip = /^\s*$/
}: BenchmarkRunOptions = {}): Promise<void> {
// Filtering candidates by the "only" and "skip" constraint
const benchmarks: BenchmarkDefinition[] = candidates.filter(
({ name }) => only.test(name) && !skip.test(name)
);
// Init main counters and error flag
const filtered = candidates.length - benchmarks.length;
let measured = 0;
let failed = false;
// Setting up a shared benchmark clock and timer
const clock: BenchmarkClock = { start: NaN, stop: NaN };
const b = createBenchmarkTimer(clock);
// Iterating given benchmark definitions (await-in-loop)
console.log(
"running",
benchmarks.length,
`benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
);
for (const { name, runs = 0, func } of benchmarks) {
// See https://github.com/denoland/deno/pull/1452 about groupCollapsed
console.groupCollapsed(`benchmark ${name} ... `);
// Trying benchmark.func
let result = "";
try {
if (runs === 1) {
// b is a benchmark timer interfacing an unset (NaN) benchmark clock
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock);
result = `${clock.stop - clock.start}ms`;
} else if (runs > 1) {
// Averaging runs
let pendingRuns = runs;
let totalMs = 0;
// Would be better 2 not run these serially
while (true) {
// b is a benchmark timer interfacing an unset (NaN) benchmark clock
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock);
// Summing up
totalMs += clock.stop - clock.start;
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
// Once all ran
if (!--pendingRuns) {
result = `${runs} runs avg: ${totalMs / runs}ms`;
break;
}
}
}
} catch (err) {
failed = true;
console.groupEnd();
console.error(red(err.stack));
break;
}
// Reporting
console.log(blue(result));
console.groupEnd();
measured++;
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
}
// Closing results
console.log(
`benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
`${measured} measured; ${filtered} filtered`
);
// Making sure the program exit code is not zero in case of failure
if (failed) {
setTimeout(() => exit(1), 0);
}
}

86
benching/readme.md
View file

@ -1,86 +0,0 @@
# benching
Basic benchmarking module. Provides flintstone millisecond resolution.
## Import
```ts
import * as benching from "https://deno.land/std/benching/mod.ts";
```
## Usage
```ts
import {
BenchmarkTimer,
runBenchmarks,
bench
} from "https://deno.land/std/benching/mod.ts";
// Simple
bench(function forIncrementX1e9(b: BenchmarkTimer) {
b.start();
for (let i = 0; i < 1e9; i++);
b.stop();
});
// Reporting average measured time for $runs runs of func
bench({
name: "runs100ForIncrementX1e6",
runs: 100,
func(b: BenchmarkTimer) {
b.start();
for (let i: number = 0; i < 1e6; i++);
b.stop();
}
});
// Itsabug
bench(function throwing(b) {
b.start();
// Throws bc the timer's stop method is never called
});
// Bench control
runBenchmarks({ skip: /throw/ });
```
## API
#### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void`
Registers a benchmark that will be run once `runBenchmarks` is called.
#### `runBenchmarks(opts?: BenchmarkRunOptions): Promise<void>`
Runs all registered benchmarks serially. Filtering can be applied by setting
`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names.
#### Other exports
```ts
/** Provides methods for starting and stopping a benchmark clock. */
export interface BenchmarkTimer {
start: () => void;
stop: () => void;
}
/** Defines a benchmark through a named function. */
export type BenchmarkFunction = {
(b: BenchmarkTimer): void | Promise<void>;
name: string;
};
/** Defines a benchmark definition with configurable runs. */
export interface BenchmarkDefinition {
func: BenchmarkFunction;
name: string;
runs?: number;
}
/** Defines runBenchmark's run constraints by matching benchmark names. */
export interface BenchmarkRunOptions {
only?: RegExp;
skip?: RegExp;
}
```

1
test.ts
View file

@ -1,6 +1,5 @@
#!/usr/bin/env deno -A
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
import "./benching/test.ts";
import "./colors/test.ts";
import "./datetime/test.ts";
import "./examples/test.ts";

74
testing/README.md
View file

@ -131,3 +131,77 @@ test(async function fails() {
});
});
```
### Benching Usage
Basic usage:
```ts
import { runBenchmarks, bench } from "https://deno.land/std/testing/bench.ts";
bench(function forIncrementX1e9(b) {
b.start();
for (let i = 0; i < 1e9; i++);
b.stop();
});
runBenchmarks();
```
Averaging execution time over multiple runs:
```ts
bench({
name: "runs100ForIncrementX1e6",
runs: 100,
func(b) {
b.start();
for (let i = 0; i < 1e6; i++);
b.stop();
}
});
```
#### Benching API
##### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void`
Registers a benchmark that will be run once `runBenchmarks` is called.
##### `runBenchmarks(opts?: BenchmarkRunOptions): Promise<void>`
Runs all registered benchmarks serially. Filtering can be applied by setting
`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names.
##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise<void>`
Runs specified benchmarks if the enclosing script is main.
##### Other exports
```ts
/** Provides methods for starting and stopping a benchmark clock. */
export interface BenchmarkTimer {
start: () => void;
stop: () => void;
}
/** Defines a benchmark through a named function. */
export interface BenchmarkFunction {
(b: BenchmarkTimer): void | Promise<void>;
name: string;
}
/** Defines a benchmark definition with configurable runs. */
export interface BenchmarkDefinition {
func: BenchmarkFunction;
name: string;
runs?: number;
}
/** Defines runBenchmark's run constraints by matching benchmark names. */
export interface BenchmarkRunOptions {
only?: RegExp;
skip?: RegExp;
}
```

189
testing/bench.ts
View file

@ -1,16 +1,179 @@
import { bench, runBenchmarks } from "./../benching/mod.ts";
import { runTests } from "./mod.ts";
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
bench(async function testingSerial(b) {
b.start();
await runTests();
b.stop();
});
const { exit, noColor } = Deno;
bench(async function testingParallel(b) {
b.start();
await runTests({ parallel: true });
b.stop();
});
interface BenchmarkClock {
start: number;
stop: number;
}
runBenchmarks({ only: /testing/ });
/** Provides methods for starting and stopping a benchmark clock. */
export interface BenchmarkTimer {
start: () => void;
stop: () => void;
}
/** Defines a benchmark through a named function. */
export interface BenchmarkFunction {
(b: BenchmarkTimer): void | Promise<void>;
name: string;
}
/** Defines a benchmark definition with configurable runs. */
export interface BenchmarkDefinition {
func: BenchmarkFunction;
name: string;
runs?: number;
}
/** Defines runBenchmark's run constraints by matching benchmark names. */
export interface BenchmarkRunOptions {
only?: RegExp;
skip?: RegExp;
}
function red(text: string): string {
return noColor ? text : `\x1b[31m${text}\x1b[0m`;
}
function blue(text: string): string {
return noColor ? text : `\x1b[34m${text}\x1b[0m`;
}
function verifyOr1Run(runs?: number): number {
return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
}
function assertTiming(clock: BenchmarkClock): void {
// NaN indicates that a benchmark has not been timed properly
if (!clock.stop) {
throw new Error("The benchmark timer's stop method must be called");
} else if (!clock.start) {
throw new Error("The benchmark timer's start method must be called");
} else if (clock.start > clock.stop) {
throw new Error(
"The benchmark timer's start method must be called before its " +
"stop method"
);
}
}
function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer {
return {
start(): void {
clock.start = Date.now();
},
stop(): void {
clock.stop = Date.now();
}
};
}
const candidates: BenchmarkDefinition[] = [];
/** Registers a benchmark as a candidate for the runBenchmarks executor. */
export function bench(
benchmark: BenchmarkDefinition | BenchmarkFunction
): void {
if (!benchmark.name) {
throw new Error("The benchmark function must not be anonymous");
}
if (typeof benchmark === "function") {
candidates.push({ name: benchmark.name, runs: 1, func: benchmark });
} else {
candidates.push({
name: benchmark.name,
runs: verifyOr1Run(benchmark.runs),
func: benchmark.func
});
}
}
/** Runs all registered and non-skipped benchmarks serially. */
export async function runBenchmarks({
only = /[^\s]/,
skip = /^\s*$/
}: BenchmarkRunOptions = {}): Promise<void> {
// Filtering candidates by the "only" and "skip" constraint
const benchmarks: BenchmarkDefinition[] = candidates.filter(
({ name }) => only.test(name) && !skip.test(name)
);
// Init main counters and error flag
const filtered = candidates.length - benchmarks.length;
let measured = 0;
let failed = false;
// Setting up a shared benchmark clock and timer
const clock: BenchmarkClock = { start: NaN, stop: NaN };
const b = createBenchmarkTimer(clock);
// Iterating given benchmark definitions (await-in-loop)
console.log(
"running",
benchmarks.length,
`benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
);
for (const { name, runs = 0, func } of benchmarks) {
// See https://github.com/denoland/deno/pull/1452 about groupCollapsed
console.groupCollapsed(`benchmark ${name} ... `);
// Trying benchmark.func
let result = "";
try {
if (runs === 1) {
// b is a benchmark timer interfacing an unset (NaN) benchmark clock
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock);
result = `${clock.stop - clock.start}ms`;
} else if (runs > 1) {
// Averaging runs
let pendingRuns = runs;
let totalMs = 0;
// Would be better 2 not run these serially
while (true) {
// b is a benchmark timer interfacing an unset (NaN) benchmark clock
await func(b);
// Making sure the benchmark was started/stopped properly
assertTiming(clock);
// Summing up
totalMs += clock.stop - clock.start;
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
// Once all ran
if (!--pendingRuns) {
result = `${runs} runs avg: ${totalMs / runs}ms`;
break;
}
}
}
} catch (err) {
failed = true;
console.groupEnd();
console.error(red(err.stack));
break;
}
// Reporting
console.log(blue(result));
console.groupEnd();
measured++;
// Resetting the benchmark clock
clock.start = clock.stop = NaN;
}
// Closing results
console.log(
`benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
`${measured} measured; ${filtered} filtered`
);
// Making sure the program exit code is not zero in case of failure
if (failed) {
setTimeout(() => exit(1), 0);
}
}
/** Runs specified benchmarks if the enclosing script is main. */
export async function runIfMain(
meta: ImportMeta,
opts?: BenchmarkRunOptions
): Promise<void> {
if (meta.main) {
return runBenchmarks(opts);
}
}

10
benching/example.ts → testing/bench_example.ts
View file

@ -1,7 +1,7 @@
// https://deno.land/std/benching/mod.ts
import { BenchmarkTimer, runBenchmarks, bench } from "./mod.ts";
// https://deno.land/std/testing/bench.ts
import { BenchmarkTimer, bench, runIfMain } from "./bench.ts";
// Simple
// Basic
bench(function forIncrementX1e9(b: BenchmarkTimer) {
b.start();
for (let i = 0; i < 1e9; i++);
@ -12,7 +12,7 @@ bench(function forIncrementX1e9(b: BenchmarkTimer) {
bench({
name: "runs100ForIncrementX1e6",
runs: 100,
func(b: BenchmarkTimer) {
func(b) {
b.start();
for (let i = 0; i < 1e6; i++);
b.stop();
@ -26,4 +26,4 @@ bench(function throwing(b) {
});
// Bench control
runBenchmarks({ skip: /throw/ });
runIfMain(import.meta, { skip: /throw/ });

20
benching/test.ts → testing/bench_test.ts
View file

@ -1,22 +1,22 @@
import { test } from "../testing/mod.ts";
import { bench, runBenchmarks, BenchmarkTimer } from "./mod.ts";
import { test, runIfMain } from "./mod.ts";
import { bench, runBenchmarks } from "./bench.ts";
import "example.ts";
import "./bench_example.ts";
test(async function benching() {
bench(function forIncrementX1e9(b: BenchmarkTimer) {
bench(function forIncrementX1e9(b) {
b.start();
for (let i = 0; i < 1e9; i++);
b.stop();
});
bench(function forDecrementX1e9(b: BenchmarkTimer) {
bench(function forDecrementX1e9(b) {
b.start();
for (let i = 1e9; i > 0; i--);
b.stop();
});
bench(async function forAwaitFetchDenolandX10(b: BenchmarkTimer) {
bench(async function forAwaitFetchDenolandX10(b) {
b.start();
for (let i = 0; i < 10; i++) {
await fetch("https://deno.land/");
@ -24,7 +24,7 @@ test(async function benching() {
b.stop();
});
bench(async function promiseAllFetchDenolandX10(b: BenchmarkTimer) {
bench(async function promiseAllFetchDenolandX10(b) {
const urls = new Array(10).fill("https://deno.land/");
b.start();
await Promise.all(urls.map((denoland: string) => fetch(denoland)));
@ -34,17 +34,19 @@ test(async function benching() {
bench({
name: "runs100ForIncrementX1e6",
runs: 100,
func(b: BenchmarkTimer) {
func(b) {
b.start();
for (let i = 0; i < 1e6; i++);
b.stop();
}
});
bench(function throwing(b: BenchmarkTimer) {
bench(function throwing(b) {
b.start();
// Throws bc the timer's stop method is never called
});
await runBenchmarks({ skip: /throw/ });
});
runIfMain(import.meta);

3
testing/test.ts
View file

@ -11,6 +11,7 @@ import "./format_test.ts";
import "./diff_test.ts";
import "./pretty_test.ts";
import "./asserts_test.ts";
import "./bench_test.ts";
test(function testingAssertEqualActualUncoercable() {
let didThrow = false;
@ -251,4 +252,4 @@ test(async function testingThrowsAsyncMsgNotIncludes() {
assert(didThrow);
});
runIfMain(import.meta, { parallel: true });
runIfMain(import.meta);

18
testing/testing_bench.ts Normal file
View file

@ -0,0 +1,18 @@
import { bench, runIfMain } from "./bench.ts";
import { runTests } from "./mod.ts";
import "./asserts_test.ts";
bench(async function testingSerial(b) {
b.start();
await runTests();
b.stop();
});
bench(async function testingParallel(b) {
b.start();
await runTests({ parallel: true });
b.stop();
});
runIfMain(import.meta);