Move benching into testing. (#258)

benching/mod.ts

@@ -1,169 +0,0 @@
-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
-const { exit, noColor } = Deno;
-
-interface BenchmarkClock {
-  start: number;
-  stop: number;
-}
-
-/** Provides methods for starting and stopping a benchmark clock. */
-export interface BenchmarkTimer {
-  start: () => void;
-  stop: () => void;
-}
-
-/** Defines a benchmark through a named function. */
-export interface BenchmarkFunction {
-  (b: BenchmarkTimer): void | Promise<void>;
-  name: string;
-}
-
-/** Defines a benchmark definition with configurable runs. */
-export interface BenchmarkDefinition {
-  func: BenchmarkFunction;
-  name: string;
-  runs?: number;
-}
-
-/** Defines runBenchmark's run constraints by matching benchmark names. */
-export interface BenchmarkRunOptions {
-  only?: RegExp;
-  skip?: RegExp;
-}
-
-function red(text: string): string {
-  return noColor ? text : `\x1b[31m${text}\x1b[0m`;
-}
-
-function blue(text: string): string {
-  return noColor ? text : `\x1b[34m${text}\x1b[0m`;
-}
-
-function verifyOr1Run(runs?: number): number {
-  return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
-}
-
-function assertTiming(clock: BenchmarkClock): void {
-  // NaN indicates that a benchmark has not been timed properly
-  if (!clock.stop) {
-    throw new Error("The benchmark timer's stop method must be called");
-  } else if (!clock.start) {
-    throw new Error("The benchmark timer's start method must be called");
-  } else if (clock.start > clock.stop) {
-    throw new Error(
-      "The benchmark timer's start method must be called before its " +
-        "stop method"
-    );
-  }
-}
-
-function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer {
-  return {
-    start(): void {
-      clock.start = Date.now();
-    },
-    stop(): void {
-      clock.stop = Date.now();
-    }
-  };
-}
-
-const candidates: BenchmarkDefinition[] = [];
-
-/** Registers a benchmark as a candidate for the runBenchmarks executor. */
-export function bench(
-  benchmark: BenchmarkDefinition | BenchmarkFunction
-): void {
-  if (!benchmark.name) {
-    throw new Error("The benchmark function must not be anonymous");
-  }
-  if (typeof benchmark === "function") {
-    candidates.push({ name: benchmark.name, runs: 1, func: benchmark });
-  } else {
-    candidates.push({
-      name: benchmark.name,
-      runs: verifyOr1Run(benchmark.runs),
-      func: benchmark.func
-    });
-  }
-}
-
-/** Runs all registered and non-skipped benchmarks serially. */
-export async function runBenchmarks({
-  only = /[^\s]/,
-  skip = /^\s*$/
-}: BenchmarkRunOptions = {}): Promise<void> {
-  // Filtering candidates by the "only" and "skip" constraint
-  const benchmarks: BenchmarkDefinition[] = candidates.filter(
-    ({ name }) => only.test(name) && !skip.test(name)
-  );
-  // Init main counters and error flag
-  const filtered = candidates.length - benchmarks.length;
-  let measured = 0;
-  let failed = false;
-  // Setting up a shared benchmark clock and timer
-  const clock: BenchmarkClock = { start: NaN, stop: NaN };
-  const b = createBenchmarkTimer(clock);
-  // Iterating given benchmark definitions (await-in-loop)
-  console.log(
-    "running",
-    benchmarks.length,
-    `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
-  );
-  for (const { name, runs = 0, func } of benchmarks) {
-    // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
-    console.groupCollapsed(`benchmark ${name} ... `);
-    // Trying benchmark.func
-    let result = "";
-    try {
-      if (runs === 1) {
-        // b is a benchmark timer interfacing an unset (NaN) benchmark clock
-        await func(b);
-        // Making sure the benchmark was started/stopped properly
-        assertTiming(clock);
-        result = `${clock.stop - clock.start}ms`;
-      } else if (runs > 1) {
-        // Averaging runs
-        let pendingRuns = runs;
-        let totalMs = 0;
-        // Would be better 2 not run these serially
-        while (true) {
-          // b is a benchmark timer interfacing an unset (NaN) benchmark clock
-          await func(b);
-          // Making sure the benchmark was started/stopped properly
-          assertTiming(clock);
-          // Summing up
-          totalMs += clock.stop - clock.start;
-          // Resetting the benchmark clock
-          clock.start = clock.stop = NaN;
-          // Once all ran
-          if (!--pendingRuns) {
-            result = `${runs} runs avg: ${totalMs / runs}ms`;
-            break;
-          }
-        }
-      }
-    } catch (err) {
-      failed = true;
-      console.groupEnd();
-      console.error(red(err.stack));
-      break;
-    }
-    // Reporting
-    console.log(blue(result));
-    console.groupEnd();
-    measured++;
-    // Resetting the benchmark clock
-    clock.start = clock.stop = NaN;
-  }
-  // Closing results
-  console.log(
-    `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
-      `${measured} measured; ${filtered} filtered`
-  );
-  // Making sure the program exit code is not zero in case of failure
-  if (failed) {
-    setTimeout(() => exit(1), 0);
-  }
-}

benching/readme.md

@@ -1,86 +0,0 @@
-# benching
-
-Basic benchmarking module. Provides flintstone millisecond resolution.
-
-## Import
-
-```ts
-import * as benching from "https://deno.land/std/benching/mod.ts";
-```
-
-## Usage
-
-```ts
-import {
-  BenchmarkTimer,
-  runBenchmarks,
-  bench
-} from "https://deno.land/std/benching/mod.ts";
-
-// Simple
-bench(function forIncrementX1e9(b: BenchmarkTimer) {
-  b.start();
-  for (let i = 0; i < 1e9; i++);
-  b.stop();
-});
-
-// Reporting average measured time for $runs runs of func
-bench({
-  name: "runs100ForIncrementX1e6",
-  runs: 100,
-  func(b: BenchmarkTimer) {
-    b.start();
-    for (let i: number = 0; i < 1e6; i++);
-    b.stop();
-  }
-});
-
-// Itsabug
-bench(function throwing(b) {
-  b.start();
-  // Throws bc the timer's stop method is never called
-});
-
-// Bench control
-runBenchmarks({ skip: /throw/ });
-```
-
-## API
-
-#### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void`
-
-Registers a benchmark that will be run once `runBenchmarks` is called.
-
-#### `runBenchmarks(opts?: BenchmarkRunOptions): Promise<void>`
-
-Runs all registered benchmarks serially. Filtering can be applied by setting
-`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names.
-
-#### Other exports
-
-```ts
-/** Provides methods for starting and stopping a benchmark clock. */
-export interface BenchmarkTimer {
-  start: () => void;
-  stop: () => void;
-}
-
-/** Defines a benchmark through a named function. */
-export type BenchmarkFunction = {
-  (b: BenchmarkTimer): void | Promise<void>;
-  name: string;
-};
-
-/** Defines a benchmark definition with configurable runs. */
-export interface BenchmarkDefinition {
-  func: BenchmarkFunction;
-  name: string;
-  runs?: number;
-}
-
-/** Defines runBenchmark's run constraints by matching benchmark names. */
-export interface BenchmarkRunOptions {
-  only?: RegExp;
-  skip?: RegExp;
-}
-```

test.ts

@@ -1,6 +1,5 @@
 #!/usr/bin/env deno -A
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import "./benching/test.ts";
 import "./colors/test.ts";
 import "./datetime/test.ts";
 import "./examples/test.ts";

testing/README.md

@@ -131,3 +131,77 @@ test(async function fails() {
   });
 });
 ```
+
+### Benching Usage
+
+Basic usage:
+
+```ts
+import { runBenchmarks, bench } from "https://deno.land/std/testing/bench.ts";
+
+bench(function forIncrementX1e9(b) {
+  b.start();
+  for (let i = 0; i < 1e9; i++);
+  b.stop();
+});
+
+runBenchmarks();
+```
+
+Averaging execution time over multiple runs:
+
+```ts
+bench({
+  name: "runs100ForIncrementX1e6",
+  runs: 100,
+  func(b) {
+    b.start();
+    for (let i = 0; i < 1e6; i++);
+    b.stop();
+  }
+});
+```
+
+#### Benching API
+
+##### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void`
+
+Registers a benchmark that will be run once `runBenchmarks` is called.
+
+##### `runBenchmarks(opts?: BenchmarkRunOptions): Promise<void>`
+
+Runs all registered benchmarks serially. Filtering can be applied by setting
+`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names.
+
+##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise<void>`
+
+Runs specified benchmarks if the enclosing script is main.
+
+##### Other exports
+
+```ts
+/** Provides methods for starting and stopping a benchmark clock. */
+export interface BenchmarkTimer {
+  start: () => void;
+  stop: () => void;
+}
+
+/** Defines a benchmark through a named function. */
+export interface BenchmarkFunction {
+  (b: BenchmarkTimer): void | Promise<void>;
+  name: string;
+}
+
+/** Defines a benchmark definition with configurable runs. */
+export interface BenchmarkDefinition {
+  func: BenchmarkFunction;
+  name: string;
+  runs?: number;
+}
+
+/** Defines runBenchmark's run constraints by matching benchmark names. */
+export interface BenchmarkRunOptions {
+  only?: RegExp;
+  skip?: RegExp;
+}
+```

testing/bench.ts

@@ -1,16 +1,179 @@
-import { bench, runBenchmarks } from "./../benching/mod.ts";
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import { runTests } from "./mod.ts";
 
-bench(async function testingSerial(b) {
+const { exit, noColor } = Deno;
-  b.start();
-  await runTests();
-  b.stop();
-});
 
-bench(async function testingParallel(b) {
+interface BenchmarkClock {
-  b.start();
+  start: number;
-  await runTests({ parallel: true });
+  stop: number;
-  b.stop();
+}
-});
 
-runBenchmarks({ only: /testing/ });
+/** Provides methods for starting and stopping a benchmark clock. */
+export interface BenchmarkTimer {
+  start: () => void;
+  stop: () => void;
+}
+
+/** Defines a benchmark through a named function. */
+export interface BenchmarkFunction {
+  (b: BenchmarkTimer): void | Promise<void>;
+  name: string;
+}
+
+/** Defines a benchmark definition with configurable runs. */
+export interface BenchmarkDefinition {
+  func: BenchmarkFunction;
+  name: string;
+  runs?: number;
+}
+
+/** Defines runBenchmark's run constraints by matching benchmark names. */
+export interface BenchmarkRunOptions {
+  only?: RegExp;
+  skip?: RegExp;
+}
+
+function red(text: string): string {
+  return noColor ? text : `\x1b[31m${text}\x1b[0m`;
+}
+
+function blue(text: string): string {
+  return noColor ? text : `\x1b[34m${text}\x1b[0m`;
+}
+
+function verifyOr1Run(runs?: number): number {
+  return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
+}
+
+function assertTiming(clock: BenchmarkClock): void {
+  // NaN indicates that a benchmark has not been timed properly
+  if (!clock.stop) {
+    throw new Error("The benchmark timer's stop method must be called");
+  } else if (!clock.start) {
+    throw new Error("The benchmark timer's start method must be called");
+  } else if (clock.start > clock.stop) {
+    throw new Error(
+      "The benchmark timer's start method must be called before its " +
+        "stop method"
+    );
+  }
+}
+
+function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer {
+  return {
+    start(): void {
+      clock.start = Date.now();
+    },
+    stop(): void {
+      clock.stop = Date.now();
+    }
+  };
+}
+
+const candidates: BenchmarkDefinition[] = [];
+
+/** Registers a benchmark as a candidate for the runBenchmarks executor. */
+export function bench(
+  benchmark: BenchmarkDefinition | BenchmarkFunction
+): void {
+  if (!benchmark.name) {
+    throw new Error("The benchmark function must not be anonymous");
+  }
+  if (typeof benchmark === "function") {
+    candidates.push({ name: benchmark.name, runs: 1, func: benchmark });
+  } else {
+    candidates.push({
+      name: benchmark.name,
+      runs: verifyOr1Run(benchmark.runs),
+      func: benchmark.func
+    });
+  }
+}
+
+/** Runs all registered and non-skipped benchmarks serially. */
+export async function runBenchmarks({
+  only = /[^\s]/,
+  skip = /^\s*$/
+}: BenchmarkRunOptions = {}): Promise<void> {
+  // Filtering candidates by the "only" and "skip" constraint
+  const benchmarks: BenchmarkDefinition[] = candidates.filter(
+    ({ name }) => only.test(name) && !skip.test(name)
+  );
+  // Init main counters and error flag
+  const filtered = candidates.length - benchmarks.length;
+  let measured = 0;
+  let failed = false;
+  // Setting up a shared benchmark clock and timer
+  const clock: BenchmarkClock = { start: NaN, stop: NaN };
+  const b = createBenchmarkTimer(clock);
+  // Iterating given benchmark definitions (await-in-loop)
+  console.log(
+    "running",
+    benchmarks.length,
+    `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
+  );
+  for (const { name, runs = 0, func } of benchmarks) {
+    // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
+    console.groupCollapsed(`benchmark ${name} ... `);
+    // Trying benchmark.func
+    let result = "";
+    try {
+      if (runs === 1) {
+        // b is a benchmark timer interfacing an unset (NaN) benchmark clock
+        await func(b);
+        // Making sure the benchmark was started/stopped properly
+        assertTiming(clock);
+        result = `${clock.stop - clock.start}ms`;
+      } else if (runs > 1) {
+        // Averaging runs
+        let pendingRuns = runs;
+        let totalMs = 0;
+        // Would be better 2 not run these serially
+        while (true) {
+          // b is a benchmark timer interfacing an unset (NaN) benchmark clock
+          await func(b);
+          // Making sure the benchmark was started/stopped properly
+          assertTiming(clock);
+          // Summing up
+          totalMs += clock.stop - clock.start;
+          // Resetting the benchmark clock
+          clock.start = clock.stop = NaN;
+          // Once all ran
+          if (!--pendingRuns) {
+            result = `${runs} runs avg: ${totalMs / runs}ms`;
+            break;
+          }
+        }
+      }
+    } catch (err) {
+      failed = true;
+      console.groupEnd();
+      console.error(red(err.stack));
+      break;
+    }
+    // Reporting
+    console.log(blue(result));
+    console.groupEnd();
+    measured++;
+    // Resetting the benchmark clock
+    clock.start = clock.stop = NaN;
+  }
+  // Closing results
+  console.log(
+    `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
+      `${measured} measured; ${filtered} filtered`
+  );
+  // Making sure the program exit code is not zero in case of failure
+  if (failed) {
+    setTimeout(() => exit(1), 0);
+  }
+}
+
+/** Runs specified benchmarks if the enclosing script is main. */
+export async function runIfMain(
+  meta: ImportMeta,
+  opts?: BenchmarkRunOptions
+): Promise<void> {
+  if (meta.main) {
+    return runBenchmarks(opts);
+  }
+}

testing/bench_example.ts

@@ -1,7 +1,7 @@
-// https://deno.land/std/benching/mod.ts
+// https://deno.land/std/testing/bench.ts
-import { BenchmarkTimer, runBenchmarks, bench } from "./mod.ts";
+import { BenchmarkTimer, bench, runIfMain } from "./bench.ts";
 
-// Simple
+// Basic
 bench(function forIncrementX1e9(b: BenchmarkTimer) {
   b.start();
   for (let i = 0; i < 1e9; i++);
@@ -12,7 +12,7 @@ bench(function forIncrementX1e9(b: BenchmarkTimer) {
 bench({
   name: "runs100ForIncrementX1e6",
   runs: 100,
-  func(b: BenchmarkTimer) {
+  func(b) {
     b.start();
     for (let i = 0; i < 1e6; i++);
     b.stop();
@@ -26,4 +26,4 @@ bench(function throwing(b) {
 });
 
 // Bench control
-runBenchmarks({ skip: /throw/ });
+runIfMain(import.meta, { skip: /throw/ });

testing/bench_test.ts

@@ -1,22 +1,22 @@
-import { test } from "../testing/mod.ts";
+import { test, runIfMain } from "./mod.ts";
-import { bench, runBenchmarks, BenchmarkTimer } from "./mod.ts";
+import { bench, runBenchmarks } from "./bench.ts";
 
-import "example.ts";
+import "./bench_example.ts";
 
 test(async function benching() {
-  bench(function forIncrementX1e9(b: BenchmarkTimer) {
+  bench(function forIncrementX1e9(b) {
     b.start();
     for (let i = 0; i < 1e9; i++);
     b.stop();
   });
 
-  bench(function forDecrementX1e9(b: BenchmarkTimer) {
+  bench(function forDecrementX1e9(b) {
     b.start();
     for (let i = 1e9; i > 0; i--);
     b.stop();
   });
 
-  bench(async function forAwaitFetchDenolandX10(b: BenchmarkTimer) {
+  bench(async function forAwaitFetchDenolandX10(b) {
     b.start();
     for (let i = 0; i < 10; i++) {
       await fetch("https://deno.land/");
@@ -24,7 +24,7 @@ test(async function benching() {
     b.stop();
   });
 
-  bench(async function promiseAllFetchDenolandX10(b: BenchmarkTimer) {
+  bench(async function promiseAllFetchDenolandX10(b) {
     const urls = new Array(10).fill("https://deno.land/");
     b.start();
     await Promise.all(urls.map((denoland: string) => fetch(denoland)));
@@ -34,17 +34,19 @@ test(async function benching() {
   bench({
     name: "runs100ForIncrementX1e6",
     runs: 100,
-    func(b: BenchmarkTimer) {
+    func(b) {
       b.start();
       for (let i = 0; i < 1e6; i++);
       b.stop();
     }
   });
 
-  bench(function throwing(b: BenchmarkTimer) {
+  bench(function throwing(b) {
     b.start();
     // Throws bc the timer's stop method is never called
   });
 
   await runBenchmarks({ skip: /throw/ });
 });
+
+runIfMain(import.meta);

testing/test.ts

@@ -11,6 +11,7 @@ import "./format_test.ts";
 import "./diff_test.ts";
 import "./pretty_test.ts";
 import "./asserts_test.ts";
+import "./bench_test.ts";
 
 test(function testingAssertEqualActualUncoercable() {
   let didThrow = false;
@@ -251,4 +252,4 @@ test(async function testingThrowsAsyncMsgNotIncludes() {
   assert(didThrow);
 });
 
-runIfMain(import.meta, { parallel: true });
+runIfMain(import.meta);

testing/testing_bench.ts

@@ -0,0 +1,18 @@
+import { bench, runIfMain } from "./bench.ts";
+import { runTests } from "./mod.ts";
+
+import "./asserts_test.ts";
+
+bench(async function testingSerial(b) {
+  b.start();
+  await runTests();
+  b.stop();
+});
+
+bench(async function testingParallel(b) {
+  b.start();
+  await runTests({ parallel: true });
+  b.stop();
+});
+
+runIfMain(import.meta);