From ce7a987009aed294d336dc420b72743ecd51db89 Mon Sep 17 00:00:00 2001 From: chiefbiiko Date: Mon, 4 Mar 2019 20:19:03 +0100 Subject: [PATCH] feat: parallel testing (denoland/deno_std#224) Original: https://github.com/denoland/deno_std/commit/41bdd096f0b300056c58a04392d109bf11c1ce8e --- testing/bench.ts | 18 ++++ testing/format_test.ts | 2 +- testing/mod.ts | 191 +++++++++++++++++++++++++++++++++++------ testing/test.ts | 6 +- 4 files changed, 189 insertions(+), 28 deletions(-) create mode 100644 testing/bench.ts diff --git a/testing/bench.ts b/testing/bench.ts new file mode 100644 index 0000000000..c5fd6b2a02 --- /dev/null +++ b/testing/bench.ts @@ -0,0 +1,18 @@ +import { bench, runBenchmarks } from "./../benching/mod.ts"; +import { runTests } from "./mod.ts"; + +import "./test.ts"; + +bench(async function testingSerial(b) { + b.start(); + await runTests(); + b.stop(); +}); + +bench(async function testingParallel(b) { + b.start(); + await runTests({ parallel: true }); + b.stop(); +}); + +runBenchmarks({ only: /testing/ }); diff --git a/testing/format_test.ts b/testing/format_test.ts index a07426046c..3e6da44802 100644 --- a/testing/format_test.ts +++ b/testing/format_test.ts @@ -56,7 +56,7 @@ test({ }); test({ - name: "prints empty arguments", + name: "prints an empty array", fn() { const val: any[] = []; assertEqual(format(val), "Array []"); diff --git a/testing/mod.ts b/testing/mod.ts index 30f2a0c984..41ae61bea3 100644 --- a/testing/mod.ts +++ b/testing/mod.ts @@ -241,55 +241,196 @@ function green_ok() { return green("ok"); } -export async function runTests() { - let passed = 0; - let failed = 0; +interface TestStats { + filtered: number; + ignored: number; + measured: number; + passed: number; + failed: number; +} - console.log("running", tests.length, "tests"); - for (let i = 0; i < tests.length; i++) { - const { fn, name } = tests[i]; - let result = green_ok(); +interface TestResult { + name: string; + error: Error; + ok: boolean; + printed: boolean; +} + +interface TestResults { + keys: Map; + cases: Map; +} + +function createTestResults(tests: Array): TestResults { + return tests.reduce( + (acc: TestResults, { name }: TestDefinition, i: number): TestResults => { + acc.keys.set(name, i); + acc.cases.set(i, { name, printed: false, ok: false, error: null }); + return acc; + }, + { cases: new Map(), keys: new Map() } + ); +} + +function report(result: TestResult): void { + if (result.ok) { + console.log(`test ${result.name} ... ${green_ok()}`); + } else if (result.error) { + console.error( + `test ${result.name} ... ${red_failed()}\n${result.error.stack}` + ); + } else { + console.log(`test ${result.name} ... unresolved`); + } + result.printed = true; +} + +function printResults( + stats: TestStats, + results: TestResults, + flush: boolean +): void { + if (flush) { + for (const result of results.cases.values()) { + if (!result.printed) { + report(result); + if (result.error && exitOnFail) { + break; + } + } + } + } + // Attempting to match the output of Rust's test runner. + console.log( + `\ntest result: ${stats.failed ? red_failed() : green_ok()}. ` + + `${stats.passed} passed; ${stats.failed} failed; ` + + `${stats.ignored} ignored; ${stats.measured} measured; ` + + `${stats.filtered} filtered out\n` + ); +} + +function previousPrinted(name: string, results: TestResults): boolean { + const curIndex: number = results.keys.get(name); + if (curIndex === 0) { + return true; + } + return results.cases.get(curIndex - 1).printed; +} + +async function createTestCase( + stats: TestStats, + results: TestResults, + { fn, name }: TestDefinition +): Promise { + const result: TestResult = results.cases.get(results.keys.get(name)); + try { + await fn(); + stats.passed++; + result.ok = true; + } catch (err) { + stats.failed++; + result.error = err; + if (exitOnFail) { + throw err; + } + } + if (previousPrinted(name, results)) { + report(result); + } +} + +function initTestCases( + stats: TestStats, + results: TestResults, + tests: Array +): Array> { + return tests.map(createTestCase.bind(null, stats, results)); +} + +async function runTestsParallel( + stats: TestStats, + results: TestResults, + tests: Array +): Promise { + try { + await Promise.all(initTestCases(stats, results, tests)); + } catch (_) { + // The error was thrown to stop awaiting all promises if exitOnFail === true + // stats.failed has been incremented and the error stored in results + } +} + +async function runTestsSerial( + stats: TestStats, + tests: Array +): Promise { + for (const { fn, name } of tests) { // See https://github.com/denoland/deno/pull/1452 // about this usage of groupCollapsed console.groupCollapsed(`test ${name} `); try { await fn(); - passed++; - console.log("...", result); + stats.passed++; + console.log("...", green_ok()); console.groupEnd(); - } catch (e) { - result = red_failed(); - console.log("...", result); + } catch (err) { + console.log("...", red_failed()); console.groupEnd(); - console.error(e); - failed++; + console.error(err.stack); + stats.failed++; if (exitOnFail) { break; } } } +} - // Attempting to match the output of Rust's test runner. - const result = failed > 0 ? red_failed() : green_ok(); - console.log( - `\ntest result: ${result}. ${passed} passed; ${failed} failed; ` + - `${ignored} ignored; ${measured} measured; ${filtered} filtered out\n` - ); +/** Defines options for controlling execution details of a test suite. */ +export interface RunOptions { + parallel?: boolean; +} - if (failed === 0) { - // All good. +/** + * Runs specified test cases. + * Parallel execution can be enabled via the boolean option; default: serial. + */ +export async function runTests({ parallel = false }: RunOptions = {}): Promise< + void +> { + const stats: TestStats = { + measured: 0, + ignored: 0, + filtered: filtered, + passed: 0, + failed: 0 + }; + const results: TestResults = createTestResults(tests); + console.log(`running ${tests.length} tests`); + if (parallel) { + await runTestsParallel(stats, results, tests); } else { + await runTestsSerial(stats, tests); + } + printResults(stats, results, parallel); + if (stats.failed) { // Use setTimeout to avoid the error being ignored due to unhandled // promise rejections being swallowed. setTimeout(() => { - console.error(`There were ${failed} test failures.`); + console.error(`There were ${stats.failed} test failures.`); Deno.exit(1); }, 0); } } -export async function runIfMain(meta: ImportMeta) { +/** + * Runs specified test cases if the enclosing script is main. + * Execution mode is toggleable via opts.parallel, defaults to false. + */ +export async function runIfMain( + meta: ImportMeta, + opts?: RunOptions +): Promise { if (meta.main) { - runTests(); + return runTests(opts); } } diff --git a/testing/test.ts b/testing/test.ts index 8c780fdf15..0d79c22184 100644 --- a/testing/test.ts +++ b/testing/test.ts @@ -1,6 +1,6 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -import { test, assert, assertEqual, equal } from "./mod.ts"; +import { test, assert, assertEqual, equal, runIfMain } from "./mod.ts"; import "./format_test.ts"; import "./diff_test.ts"; import "./pretty_test.ts"; @@ -267,7 +267,7 @@ test(async function testingThrowsAsyncMsgIncludes() { assert(count === 1); }); -test(async function testingThrowsMsgNotIncludes() { +test(async function testingThrowsAsyncMsgNotIncludes() { let count = 0; let didThrow = false; try { @@ -289,3 +289,5 @@ test(async function testingThrowsMsgNotIncludes() { assert(count === 1); assert(didThrow); }); + +runIfMain(import.meta, { parallel: true });