feat: parallel testing (denoland/deno_std#224)

Original: 41bdd096f0

testing/bench.ts

@@ -0,0 +1,18 @@
+import { bench, runBenchmarks } from "./../benching/mod.ts";
+import { runTests } from "./mod.ts";
+
+import "./test.ts";
+
+bench(async function testingSerial(b) {
+  b.start();
+  await runTests();
+  b.stop();
+});
+
+bench(async function testingParallel(b) {
+  b.start();
+  await runTests({ parallel: true });
+  b.stop();
+});
+
+runBenchmarks({ only: /testing/ });

testing/format_test.ts

@@ -56,7 +56,7 @@ test({
 });
 
 test({
-  name: "prints empty arguments",
+  name: "prints an empty array",
   fn() {
     const val: any[] = [];
     assertEqual(format(val), "Array []");

testing/mod.ts

@@ -241,55 +241,196 @@ function green_ok() {
   return green("ok");
 }
 
-export async function runTests() {
+interface TestStats {
-  let passed = 0;
+  filtered: number;
-  let failed = 0;
+  ignored: number;
+  measured: number;
+  passed: number;
+  failed: number;
+}
 
-  console.log("running", tests.length, "tests");
+interface TestResult {
-  for (let i = 0; i < tests.length; i++) {
+  name: string;
-    const { fn, name } = tests[i];
+  error: Error;
-    let result = green_ok();
+  ok: boolean;
+  printed: boolean;
+}
+
+interface TestResults {
+  keys: Map<string, number>;
+  cases: Map<number, TestResult>;
+}
+
+function createTestResults(tests: Array<TestDefinition>): TestResults {
+  return tests.reduce(
+    (acc: TestResults, { name }: TestDefinition, i: number): TestResults => {
+      acc.keys.set(name, i);
+      acc.cases.set(i, { name, printed: false, ok: false, error: null });
+      return acc;
+    },
+    { cases: new Map(), keys: new Map() }
+  );
+}
+
+function report(result: TestResult): void {
+  if (result.ok) {
+    console.log(`test ${result.name} ... ${green_ok()}`);
+  } else if (result.error) {
+    console.error(
+      `test ${result.name} ... ${red_failed()}\n${result.error.stack}`
+    );
+  } else {
+    console.log(`test ${result.name} ... unresolved`);
+  }
+  result.printed = true;
+}
+
+function printResults(
+  stats: TestStats,
+  results: TestResults,
+  flush: boolean
+): void {
+  if (flush) {
+    for (const result of results.cases.values()) {
+      if (!result.printed) {
+        report(result);
+        if (result.error && exitOnFail) {
+          break;
+        }
+      }
+    }
+  }
+  // Attempting to match the output of Rust's test runner.
+  console.log(
+    `\ntest result: ${stats.failed ? red_failed() : green_ok()}. ` +
+      `${stats.passed} passed; ${stats.failed} failed; ` +
+      `${stats.ignored} ignored; ${stats.measured} measured; ` +
+      `${stats.filtered} filtered out\n`
+  );
+}
+
+function previousPrinted(name: string, results: TestResults): boolean {
+  const curIndex: number = results.keys.get(name);
+  if (curIndex === 0) {
+    return true;
+  }
+  return results.cases.get(curIndex - 1).printed;
+}
+
+async function createTestCase(
+  stats: TestStats,
+  results: TestResults,
+  { fn, name }: TestDefinition
+): Promise<void> {
+  const result: TestResult = results.cases.get(results.keys.get(name));
+  try {
+    await fn();
+    stats.passed++;
+    result.ok = true;
+  } catch (err) {
+    stats.failed++;
+    result.error = err;
+    if (exitOnFail) {
+      throw err;
+    }
+  }
+  if (previousPrinted(name, results)) {
+    report(result);
+  }
+}
+
+function initTestCases(
+  stats: TestStats,
+  results: TestResults,
+  tests: Array<TestDefinition>
+): Array<Promise<void>> {
+  return tests.map(createTestCase.bind(null, stats, results));
+}
+
+async function runTestsParallel(
+  stats: TestStats,
+  results: TestResults,
+  tests: Array<TestDefinition>
+): Promise<void> {
+  try {
+    await Promise.all(initTestCases(stats, results, tests));
+  } catch (_) {
+    // The error was thrown to stop awaiting all promises if exitOnFail === true
+    // stats.failed has been incremented and the error stored in results
+  }
+}
+
+async function runTestsSerial(
+  stats: TestStats,
+  tests: Array<TestDefinition>
+): Promise<void> {
+  for (const { fn, name } of tests) {
     // See https://github.com/denoland/deno/pull/1452
     // about this usage of groupCollapsed
     console.groupCollapsed(`test ${name} `);
     try {
       await fn();
-      passed++;
+      stats.passed++;
-      console.log("...", result);
+      console.log("...", green_ok());
       console.groupEnd();
-    } catch (e) {
+    } catch (err) {
-      result = red_failed();
+      console.log("...", red_failed());
-      console.log("...", result);
       console.groupEnd();
-      console.error(e);
+      console.error(err.stack);
-      failed++;
+      stats.failed++;
       if (exitOnFail) {
         break;
       }
     }
   }
+}
 
-  // Attempting to match the output of Rust's test runner.
+/** Defines options for controlling execution details of a test suite. */
-  const result = failed > 0 ? red_failed() : green_ok();
+export interface RunOptions {
-  console.log(
+  parallel?: boolean;
-    `\ntest result: ${result}. ${passed} passed; ${failed} failed; ` +
+}
-      `${ignored} ignored; ${measured} measured; ${filtered} filtered out\n`
-  );
 
-  if (failed === 0) {
+/**
-    // All good.
+ * Runs specified test cases.
+ * Parallel execution can be enabled via the boolean option; default: serial.
+ */
+export async function runTests({ parallel = false }: RunOptions = {}): Promise<
+  void
+> {
+  const stats: TestStats = {
+    measured: 0,
+    ignored: 0,
+    filtered: filtered,
+    passed: 0,
+    failed: 0
+  };
+  const results: TestResults = createTestResults(tests);
+  console.log(`running ${tests.length} tests`);
+  if (parallel) {
+    await runTestsParallel(stats, results, tests);
   } else {
+    await runTestsSerial(stats, tests);
+  }
+  printResults(stats, results, parallel);
+  if (stats.failed) {
     // Use setTimeout to avoid the error being ignored due to unhandled
     // promise rejections being swallowed.
     setTimeout(() => {
-      console.error(`There were ${failed} test failures.`);
+      console.error(`There were ${stats.failed} test failures.`);
       Deno.exit(1);
     }, 0);
   }
 }
 
-export async function runIfMain(meta: ImportMeta) {
+/**
+ * Runs specified test cases if the enclosing script is main.
+ * Execution mode is toggleable via opts.parallel, defaults to false.
+ */
+export async function runIfMain(
+  meta: ImportMeta,
+  opts?: RunOptions
+): Promise<void> {
   if (meta.main) {
-    runTests();
+    return runTests(opts);
   }
 }

testing/test.ts

@@ -1,6 +1,6 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
 
-import { test, assert, assertEqual, equal } from "./mod.ts";
+import { test, assert, assertEqual, equal, runIfMain } from "./mod.ts";
 import "./format_test.ts";
 import "./diff_test.ts";
 import "./pretty_test.ts";
@@ -267,7 +267,7 @@ test(async function testingThrowsAsyncMsgIncludes() {
   assert(count === 1);
 });
 
-test(async function testingThrowsMsgNotIncludes() {
+test(async function testingThrowsAsyncMsgNotIncludes() {
   let count = 0;
   let didThrow = false;
   try {
@@ -289,3 +289,5 @@ test(async function testingThrowsMsgNotIncludes() {
   assert(count === 1);
   assert(didThrow);
 });
+
+runIfMain(import.meta, { parallel: true });