From 426ebf854a82c63cdaa2413fbd1b005025dba95b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 11 Oct 2021 09:45:02 -0400 Subject: [PATCH] feat(unstable/test): imperative test steps API (#12190) --- cli/dts/lib.deno.ns.d.ts | 12 +- cli/dts/lib.deno.unstable.d.ts | 37 ++ cli/tests/integration/test_tests.rs | 36 ++ .../testdata/test/steps/failing_steps.out | 53 ++ .../testdata/test/steps/failing_steps.ts | 27 + .../testdata/test/steps/ignored_steps.out | 8 + .../testdata/test/steps/ignored_steps.ts | 16 + .../testdata/test/steps/invalid_usage.out | 111 ++++ .../testdata/test/steps/invalid_usage.ts | 122 +++++ .../testdata/test/steps/no_unstable_flag.out | 13 + .../testdata/test/steps/no_unstable_flag.ts | 4 + .../testdata/test/steps/passing_steps.out | 38 ++ .../testdata/test/steps/passing_steps.ts | 120 +++++ cli/tests/unit/test_util.ts | 2 +- cli/tests/unit/testing_test.ts | 40 +- cli/tools/test.rs | 175 +++++- runtime/js/40_testing.js | 508 ++++++++++++++++-- runtime/js/99_main.js | 3 + 18 files changed, 1279 insertions(+), 46 deletions(-) create mode 100644 cli/tests/testdata/test/steps/failing_steps.out create mode 100644 cli/tests/testdata/test/steps/failing_steps.ts create mode 100644 cli/tests/testdata/test/steps/ignored_steps.out create mode 100644 cli/tests/testdata/test/steps/ignored_steps.ts create mode 100644 cli/tests/testdata/test/steps/invalid_usage.out create mode 100644 cli/tests/testdata/test/steps/invalid_usage.ts create mode 100644 cli/tests/testdata/test/steps/no_unstable_flag.out create mode 100644 cli/tests/testdata/test/steps/no_unstable_flag.ts create mode 100644 cli/tests/testdata/test/steps/passing_steps.out create mode 100644 cli/tests/testdata/test/steps/passing_steps.ts diff --git a/cli/dts/lib.deno.ns.d.ts b/cli/dts/lib.deno.ns.d.ts index 4d3dfe0d31..eb91d6fa46 100644 --- a/cli/dts/lib.deno.ns.d.ts +++ b/cli/dts/lib.deno.ns.d.ts @@ -113,8 +113,12 @@ declare namespace Deno { * See: https://no-color.org/ */ export const noColor: boolean; + /** **UNSTABLE**: New option, yet to be vetted. */ + export interface TestContext { + } + export interface TestDefinition { - fn: () => void | Promise; + fn: (t: TestContext) => void | Promise; name: string; ignore?: boolean; /** If at least one test has `only` set to true, only run tests that have @@ -127,7 +131,6 @@ declare namespace Deno { * after the test has exactly the same contents as before the test. Defaults * to true. */ sanitizeResources?: boolean; - /** Ensure the test case does not prematurely cause the process to exit, * for example via a call to `Deno.exit`. Defaults to true. */ sanitizeExit?: boolean; @@ -184,7 +187,10 @@ declare namespace Deno { * }); * ``` */ - export function test(name: string, fn: () => void | Promise): void; + export function test( + name: string, + fn: (t: TestContext) => void | Promise, + ): void; /** Exit the Deno process with optional exit code. If no exit code is supplied * then Deno will exit with return code of 0. diff --git a/cli/dts/lib.deno.unstable.d.ts b/cli/dts/lib.deno.unstable.d.ts index 73f4bfcb24..3bea165e51 100644 --- a/cli/dts/lib.deno.unstable.d.ts +++ b/cli/dts/lib.deno.unstable.d.ts @@ -948,6 +948,43 @@ declare namespace Deno { }; } + /** **UNSTABLE**: New option, yet to be vetted. */ + export interface TestContext { + /** Run a sub step of the parent test with a given name. Returns a promise + * that resolves to a boolean signifying if the step completed successfully. + * The returned promise never rejects unless the arguments are invalid. + * If the test was ignored, the promise returns `false`. + */ + step(t: TestStepDefinition): Promise; + + /** Run a sub step of the parent test with a given name. Returns a promise + * that resolves to a boolean signifying if the step completed successfully. + * The returned promise never rejects unless the arguments are invalid. + * If the test was ignored, the promise returns `false`. + */ + step( + name: string, + fn: (t: TestContext) => void | Promise, + ): Promise; + } + + /** **UNSTABLE**: New option, yet to be vetted. */ + export interface TestStepDefinition { + fn: (t: TestContext) => void | Promise; + name: string; + ignore?: boolean; + /** Check that the number of async completed ops after the test is the same + * as number of dispatched ops. Defaults to true. */ + sanitizeOps?: boolean; + /** Ensure the test case does not "leak" resources - ie. the resource table + * after the test has exactly the same contents as before the test. Defaults + * to true. */ + sanitizeResources?: boolean; + /** Ensure the test case does not prematurely cause the process to exit, + * for example via a call to `Deno.exit`. Defaults to true. */ + sanitizeExit?: boolean; + } + /** **UNSTABLE**: new API, yet to be vetted. * * A generic transport listener for message-oriented protocols. */ diff --git a/cli/tests/integration/test_tests.rs b/cli/tests/integration/test_tests.rs index 24ceeefb4c..3ea8186b87 100644 --- a/cli/tests/integration/test_tests.rs +++ b/cli/tests/integration/test_tests.rs @@ -186,3 +186,39 @@ itest!(aggregate_error { exit_code: 1, output: "test/aggregate_error.out", }); + +itest!(steps_passing_steps { + args: "test --unstable test/steps/passing_steps.ts", + exit_code: 0, + output: "test/steps/passing_steps.out", +}); + +itest!(steps_passing_steps_concurrent { + args: "test --unstable --jobs=2 test/steps/passing_steps.ts", + exit_code: 0, + output: "test/steps/passing_steps.out", +}); + +itest!(steps_failing_steps { + args: "test --unstable test/steps/failing_steps.ts", + exit_code: 1, + output: "test/steps/failing_steps.out", +}); + +itest!(steps_ignored_steps { + args: "test --unstable test/steps/ignored_steps.ts", + exit_code: 0, + output: "test/steps/ignored_steps.out", +}); + +itest!(steps_invalid_usage { + args: "test --unstable test/steps/invalid_usage.ts", + exit_code: 1, + output: "test/steps/invalid_usage.out", +}); + +itest!(steps_no_unstable_flag { + args: "test test/steps/no_unstable_flag.ts", + exit_code: 1, + output: "test/steps/no_unstable_flag.out", +}); diff --git a/cli/tests/testdata/test/steps/failing_steps.out b/cli/tests/testdata/test/steps/failing_steps.out new file mode 100644 index 0000000000..1c5e2e5915 --- /dev/null +++ b/cli/tests/testdata/test/steps/failing_steps.out @@ -0,0 +1,53 @@ +[WILDCARD] +running 3 tests from [WILDCARD]/failing_steps.ts +test nested failure ... + test step 1 ... + test inner 1 ... FAILED ([WILDCARD]) + Error: Failed. + at [WILDCARD]/failing_steps.ts:[WILDCARD] + [WILDCARD] + test inner 2 ... ok ([WILDCARD]) + FAILED ([WILDCARD]) +FAILED ([WILDCARD]) +test multiple test step failures ... + test step 1 ... FAILED ([WILDCARD]) + Error: Fail. + [WILDCARD] + test step 2 ... FAILED ([WILDCARD]) + Error: Fail. + at [WILDCARD]/failing_steps.ts:[WILDCARD] + [WILDCARD] +FAILED ([WILDCARD]) +test failing step in failing test ... + test step 1 ... FAILED ([WILDCARD]) + Error: Fail. + at [WILDCARD]/failing_steps.ts:[WILDCARD] + at [WILDCARD] +FAILED ([WILDCARD]) + +failures: + +nested failure +Error: 1 test step failed. + at runTest (deno:runtime/js/40_testing.js:[WILDCARD]) + at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD]) + +multiple test step failures +Error: 2 test steps failed. + at runTest (deno:runtime/js/40_testing.js:[WILDCARD]) + at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD]) + +failing step in failing test +Error: Fail test. + at [WILDCARD]/failing_steps.ts:[WILDCARD] + at [WILDCARD] + +failures: + + nested failure + multiple test step failures + failing step in failing test + +test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD]) + +error: Test failed diff --git a/cli/tests/testdata/test/steps/failing_steps.ts b/cli/tests/testdata/test/steps/failing_steps.ts new file mode 100644 index 0000000000..efa18d54ef --- /dev/null +++ b/cli/tests/testdata/test/steps/failing_steps.ts @@ -0,0 +1,27 @@ +Deno.test("nested failure", async (t) => { + const success = await t.step("step 1", async (t) => { + let success = await t.step("inner 1", () => { + throw new Error("Failed."); + }); + if (success) throw new Error("Expected failure"); + + success = await t.step("inner 2", () => {}); + if (!success) throw new Error("Expected success"); + }); + + if (success) throw new Error("Expected failure"); +}); + +Deno.test("multiple test step failures", async (t) => { + await t.step("step 1", () => { + throw new Error("Fail."); + }); + await t.step("step 2", () => Promise.reject(new Error("Fail."))); +}); + +Deno.test("failing step in failing test", async (t) => { + await t.step("step 1", () => { + throw new Error("Fail."); + }); + throw new Error("Fail test."); +}); diff --git a/cli/tests/testdata/test/steps/ignored_steps.out b/cli/tests/testdata/test/steps/ignored_steps.out new file mode 100644 index 0000000000..c667a3d959 --- /dev/null +++ b/cli/tests/testdata/test/steps/ignored_steps.out @@ -0,0 +1,8 @@ +[WILDCARD] +running 1 test from [WILDCARD]/ignored_steps.ts +test ignored step ... + test step 1 ... ignored ([WILDCARD]) + test step 2 ... ok ([WILDCARD]) +ok ([WILDCARD]) + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD] diff --git a/cli/tests/testdata/test/steps/ignored_steps.ts b/cli/tests/testdata/test/steps/ignored_steps.ts new file mode 100644 index 0000000000..102b481fba --- /dev/null +++ b/cli/tests/testdata/test/steps/ignored_steps.ts @@ -0,0 +1,16 @@ +Deno.test("ignored step", async (t) => { + let result = await t.step({ + name: "step 1", + ignore: true, + fn: () => { + throw new Error("Fail."); + }, + }); + if (result !== false) throw new Error("Expected false."); + result = await t.step({ + name: "step 2", + ignore: false, + fn: () => {}, + }); + if (result !== true) throw new Error("Expected true."); +}); diff --git a/cli/tests/testdata/test/steps/invalid_usage.out b/cli/tests/testdata/test/steps/invalid_usage.out new file mode 100644 index 0000000000..b03ca57b60 --- /dev/null +++ b/cli/tests/testdata/test/steps/invalid_usage.out @@ -0,0 +1,111 @@ +[WILDCARD] +running 7 tests from [WILDCARD]/invalid_usage.ts +test capturing ... + test some step ... ok ([WILDCARD]) +FAILED ([WILDCARD]) +test top level missing await ... + test step ... pending ([WILDCARD]) +FAILED ([WILDCARD]) +test inner missing await ... + test step ... + test inner ... pending ([WILDCARD]) + Error: Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). + at postValidation [WILDCARD] + at testStepSanitizer [WILDCARD] + FAILED ([WILDCARD]) + Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). + at postValidation [WILDCARD] + at testStepSanitizer [WILDCARD] + at async fn ([WILDCARD]/invalid_usage.ts:[WILDCARD]) + at async Object.testStepSanitizer [WILDCARD] +FAILED ([WILDCARD]) +test parallel steps with sanitizers ... + test step 1 ... pending ([WILDCARD]) + test step 2 ... FAILED ([WILDCARD]) + Error: Cannot start test step while another test step with sanitizers is running. + * parallel steps with sanitizers > step 1 + at preValidation ([WILDCARD]) + at testStepSanitizer ([WILDCARD]) + at [WILDCARD]/invalid_usage.ts:[WILDCARD] + at [WILDCARD] +FAILED ([WILDCARD]) +test parallel steps when first has sanitizer ... + test step 1 ... pending ([WILDCARD]) + test step 2 ... FAILED ([WILDCARD]) + Error: Cannot start test step while another test step with sanitizers is running. + * parallel steps when first has sanitizer > step 1 + at preValidation ([WILDCARD]) + at testStepSanitizer ([WILDCARD]) + at [WILDCARD]/invalid_usage.ts:[WILDCARD] + at [WILDCARD] +FAILED ([WILDCARD]) +test parallel steps when second has sanitizer ... + test step 1 ... ok ([WILDCARD]) + test step 2 ... FAILED ([WILDCARD]) + Error: Cannot start test step with sanitizers while another test step is running. + * parallel steps when second has sanitizer > step 1 + at preValidation ([WILDCARD]) + at testStepSanitizer ([WILDCARD]) + at [WILDCARD]/invalid_usage.ts:[WILDCARD] + at [WILDCARD] +FAILED ([WILDCARD]) +test parallel steps where only inner tests have sanitizers ... + test step 1 ... + test step inner ... ok ([WILDCARD]) + ok ([WILDCARD]) + test step 2 ... + test step inner ... FAILED ([WILDCARD]) + Error: Cannot start test step with sanitizers while another test step is running. + * parallel steps where only inner tests have sanitizers > step 1 + at preValidation ([WILDCARD]) + at testStepSanitizer ([WILDCARD]) + at [WILDCARD]/invalid_usage.ts:[WILDCARD] + FAILED ([WILDCARD]) +FAILED ([WILDCARD]) + +failures: + +capturing +Error: Cannot run test step after parent scope has finished execution. Ensure any `.step(...)` calls are executed before their parent scope completes execution. + at TestContext.step ([WILDCARD]) + at [WILDCARD]/invalid_usage.ts:[WILDCARD] + at [WILDCARD] + +top level missing await +Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). + at postValidation [WILDCARD] + at testStepSanitizer ([WILDCARD]) + [WILDCARD] + +inner missing await +Error: 1 test step failed. + at [WILDCARD] + +parallel steps with sanitizers +Error: 1 test step failed. + at runTest ([WILDCARD]) + at [WILDCARD] + +parallel steps when first has sanitizer +Error: 1 test step failed. + at runTest ([WILDCARD]) + at [WILDCARD] + +parallel steps when second has sanitizer +Error: 1 test step failed. + at runTest ([WILDCARD]) + at [WILDCARD] + +failures: + + capturing + top level missing await + inner missing await + parallel steps with sanitizers + parallel steps when first has sanitizer + parallel steps when second has sanitizer + parallel steps where only inner tests have sanitizers + +test result: FAILED. 0 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD]) + +error: Test failed diff --git a/cli/tests/testdata/test/steps/invalid_usage.ts b/cli/tests/testdata/test/steps/invalid_usage.ts new file mode 100644 index 0000000000..f670c842e4 --- /dev/null +++ b/cli/tests/testdata/test/steps/invalid_usage.ts @@ -0,0 +1,122 @@ +import { deferred } from "../../../../../test_util/std/async/deferred.ts"; + +Deno.test("capturing", async (t) => { + let capturedContext!: Deno.TestContext; + await t.step("some step", (t) => { + capturedContext = t; + }); + // this should error because the scope of the tester has already completed + await capturedContext.step("next step", () => {}); +}); + +Deno.test("top level missing await", (t) => { + t.step("step", () => { + return new Promise((resolve) => setTimeout(resolve, 10)); + }); +}); + +Deno.test({ + name: "inner missing await", + fn: async (t) => { + await t.step("step", (t) => { + t.step("inner", () => { + return new Promise((resolve) => setTimeout(resolve, 10)); + }); + }); + await new Promise((resolve) => setTimeout(resolve, 10)); + }, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, +}); + +Deno.test("parallel steps with sanitizers", async (t) => { + // not allowed because steps with sanitizers cannot be run in parallel + const step1Entered = deferred(); + const step2Finished = deferred(); + const step1 = t.step("step 1", async () => { + step1Entered.resolve(); + await step2Finished; + }); + await step1Entered; + await t.step("step 2", () => {}); + step2Finished.resolve(); + await step1; +}); + +Deno.test("parallel steps when first has sanitizer", async (t) => { + const step1Entered = deferred(); + const step2Finished = deferred(); + const step1 = t.step({ + name: "step 1", + fn: async () => { + step1Entered.resolve(); + await step2Finished; + }, + }); + await step1Entered; + await t.step({ + name: "step 2", + fn: () => {}, + sanitizeOps: false, + sanitizeResources: false, + sanitizeExit: false, + }); + step2Finished.resolve(); + await step1; +}); + +Deno.test("parallel steps when second has sanitizer", async (t) => { + const step1Entered = deferred(); + const step2Finished = deferred(); + const step1 = t.step({ + name: "step 1", + fn: async () => { + step1Entered.resolve(); + await step2Finished; + }, + sanitizeOps: false, + sanitizeResources: false, + sanitizeExit: false, + }); + await step1Entered; + await t.step({ + name: "step 2", + fn: async () => { + await new Promise((resolve) => setTimeout(resolve, 100)); + }, + }); + step2Finished.resolve(); + await step1; +}); + +Deno.test({ + name: "parallel steps where only inner tests have sanitizers", + fn: async (t) => { + const step1Entered = deferred(); + const step2Finished = deferred(); + const step1 = t.step("step 1", async (t) => { + await t.step({ + name: "step inner", + fn: async () => { + step1Entered.resolve(); + await step2Finished; + }, + sanitizeOps: true, + }); + }); + await step1Entered; + await t.step("step 2", async (t) => { + await t.step({ + name: "step inner", + fn: () => {}, + sanitizeOps: true, + }); + }); + step2Finished.resolve(); + await step1; + }, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, +}); diff --git a/cli/tests/testdata/test/steps/no_unstable_flag.out b/cli/tests/testdata/test/steps/no_unstable_flag.out new file mode 100644 index 0000000000..8fe6ba4f7c --- /dev/null +++ b/cli/tests/testdata/test/steps/no_unstable_flag.out @@ -0,0 +1,13 @@ +[WILDCARD] +running 1 test from [WILDCARD]/no_unstable_flag.ts +test description ... FAILED ([WILDCARD]) + +failures: + +description +Error: Test steps are unstable. The --unstable flag must be provided. + at [WILDCARD] + +failures: + +[WILDCARD] diff --git a/cli/tests/testdata/test/steps/no_unstable_flag.ts b/cli/tests/testdata/test/steps/no_unstable_flag.ts new file mode 100644 index 0000000000..737efba117 --- /dev/null +++ b/cli/tests/testdata/test/steps/no_unstable_flag.ts @@ -0,0 +1,4 @@ +Deno.test("description", async (t) => { + // deno-lint-ignore no-explicit-any + await (t as any).step("step", () => {}); +}); diff --git a/cli/tests/testdata/test/steps/passing_steps.out b/cli/tests/testdata/test/steps/passing_steps.out new file mode 100644 index 0000000000..b92327d173 --- /dev/null +++ b/cli/tests/testdata/test/steps/passing_steps.out @@ -0,0 +1,38 @@ +[WILDCARD] +running 5 tests from [WILDCARD] +test description ... + test step 1 ... + test inner 1 ... ok ([WILDCARD]ms) + test inner 2 ... ok ([WILDCARD]ms) + ok ([WILDCARD]ms) +ok ([WILDCARD]ms) +test parallel steps without sanitizers ... + test step 1 ... ok ([WILDCARD]) + test step 2 ... ok ([WILDCARD]) +ok ([WILDCARD]) +test parallel steps without sanitizers due to parent ... + test step 1 ... ok ([WILDCARD]) + test step 2 ... ok ([WILDCARD]) +ok ([WILDCARD]) +test steps with disabled sanitizers, then enabled, then parallel disabled ... + test step 1 ... + test step 1 ... + test step 1 ... + test step 1 ... ok ([WILDCARD]) + test step 1 ... ok ([WILDCARD]) + ok ([WILDCARD]) + test step 2 ... ok ([WILDCARD]) + ok ([WILDCARD]) + ok ([WILDCARD]) +ok ([WILDCARD]) +test steps buffered then streaming reporting ... + test step 1 ... + test step 1 - 1 ... ok ([WILDCARD]) + test step 1 - 2 ... + test step 1 - 2 - 1 ... ok ([WILDCARD]) + ok ([WILDCARD]) + ok ([WILDCARD]) + test step 2 ... ok ([WILDCARD]) +ok ([WILDCARD]) + +test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD] diff --git a/cli/tests/testdata/test/steps/passing_steps.ts b/cli/tests/testdata/test/steps/passing_steps.ts new file mode 100644 index 0000000000..fbd52e2d30 --- /dev/null +++ b/cli/tests/testdata/test/steps/passing_steps.ts @@ -0,0 +1,120 @@ +import { deferred } from "../../../../../test_util/std/async/deferred.ts"; + +Deno.test("description", async (t) => { + const success = await t.step("step 1", async (t) => { + await t.step("inner 1", () => {}); + await t.step("inner 2", () => {}); + }); + + if (!success) throw new Error("Expected the step to return true."); +}); + +Deno.test("parallel steps without sanitizers", async (t) => { + // allowed + await Promise.all([ + t.step({ + name: "step 1", + fn: async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }, + sanitizeOps: false, + sanitizeResources: false, + sanitizeExit: false, + }), + t.step({ + name: "step 2", + fn: async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }, + sanitizeOps: false, + sanitizeResources: false, + sanitizeExit: false, + }), + ]); +}); + +Deno.test({ + name: "parallel steps without sanitizers due to parent", + fn: async (t) => { + // allowed because parent disabled the sanitizers + await Promise.all([ + t.step("step 1", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }), + t.step("step 2", async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + }), + ]); + }, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, +}); + +Deno.test({ + name: "steps with disabled sanitizers, then enabled, then parallel disabled", + fn: async (t) => { + await t.step("step 1", async (t) => { + await t.step({ + name: "step 1", + fn: async (t) => { + await Promise.all([ + t.step({ + name: "step 1", + fn: async (t) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + await Promise.all([ + t.step("step 1", () => {}), + t.step("step 1", () => {}), + ]); + }, + sanitizeExit: false, + sanitizeResources: false, + sanitizeOps: false, + }), + t.step({ + name: "step 2", + fn: () => {}, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, + }), + ]); + }, + sanitizeResources: true, + sanitizeOps: true, + sanitizeExit: true, + }); + }); + }, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, +}); + +Deno.test("steps buffered then streaming reporting", async (t) => { + // no sanitizers so this will be buffered + await t.step({ + name: "step 1", + fn: async (t) => { + // also ensure the buffered tests display in order regardless of the second one finishing first + const step2Finished = deferred(); + const step1 = t.step("step 1 - 1", async () => { + await step2Finished; + }); + const step2 = t.step("step 1 - 2", async (t) => { + await t.step("step 1 - 2 - 1", () => {}); + }); + await step2; + step2Finished.resolve(); + await step1; + }, + sanitizeResources: false, + sanitizeOps: false, + sanitizeExit: false, + }); + + // now this will start streaming and we want to + // ensure it flushes the buffer of the last test + await t.step("step 2", async () => {}); +}); diff --git a/cli/tests/unit/test_util.ts b/cli/tests/unit/test_util.ts index ee924fe8ad..65d23af650 100644 --- a/cli/tests/unit/test_util.ts +++ b/cli/tests/unit/test_util.ts @@ -39,7 +39,7 @@ interface UnitTestOptions { permissions?: UnitTestPermissions; } -type TestFunction = () => void | Promise; +type TestFunction = (tester: Deno.TestContext) => void | Promise; export function unitTest(fn: TestFunction): void; export function unitTest(options: UnitTestOptions, fn: TestFunction): void; diff --git a/cli/tests/unit/testing_test.ts b/cli/tests/unit/testing_test.ts index 89b3cc31fb..144246002a 100644 --- a/cli/tests/unit/testing_test.ts +++ b/cli/tests/unit/testing_test.ts @@ -1,5 +1,5 @@ // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. -import { assertThrows, unitTest } from "./test_util.ts"; +import { assertRejects, assertThrows, unitTest } from "./test_util.ts"; unitTest(function testFnOverloading() { // just verifying that you can use this test definition syntax @@ -25,3 +25,41 @@ unitTest(function nameOfTestCaseCantBeEmpty() { "The test name can't be empty", ); }); + +unitTest(function invalidStepArguments(t) { + assertRejects( + async () => { + // deno-lint-ignore no-explicit-any + await (t as any).step("test"); + }, + TypeError, + "Expected function for second argument.", + ); + + assertRejects( + async () => { + // deno-lint-ignore no-explicit-any + await (t as any).step("test", "not a function"); + }, + TypeError, + "Expected function for second argument.", + ); + + assertRejects( + async () => { + // deno-lint-ignore no-explicit-any + await (t as any).step(); + }, + TypeError, + "Expected a test definition or name and function.", + ); + + assertRejects( + async () => { + // deno-lint-ignore no-explicit-any + await (t as any).step(() => {}); + }, + TypeError, + "Expected a test definition or name and function.", + ); +}); diff --git a/cli/tools/test.rs b/cli/tools/test.rs index e14b5cc8b1..aec6e68567 100644 --- a/cli/tools/test.rs +++ b/cli/tools/test.rs @@ -39,7 +39,9 @@ use rand::seq::SliceRandom; use rand::SeedableRng; use regex::Regex; use serde::Deserialize; +use std::collections::HashMap; use std::collections::HashSet; +use std::io::Write; use std::num::NonZeroUsize; use std::path::PathBuf; use std::sync::mpsc::channel; @@ -60,7 +62,7 @@ enum TestMode { Both, } -#[derive(Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)] #[serde(rename_all = "camelCase")] pub struct TestDescription { pub origin: String, @@ -82,6 +84,33 @@ pub enum TestResult { Failed(String), } +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct TestStepDescription { + pub test: TestDescription, + pub level: usize, + pub name: String, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TestStepResult { + Ok, + Ignored, + Failed(Option), + Pending(Option), +} + +impl TestStepResult { + fn error(&self) -> Option<&str> { + match self { + TestStepResult::Failed(Some(text)) => Some(text.as_str()), + TestStepResult::Pending(Some(text)) => Some(text.as_str()), + _ => None, + } + } +} + #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TestPlan { @@ -98,6 +127,8 @@ pub enum TestEvent { Wait(TestDescription), Output(TestOutput), Result(TestDescription, TestResult, u64), + StepWait(TestStepDescription), + StepResult(TestStepDescription, TestStepResult, u64), } #[derive(Debug, Clone, Deserialize)] @@ -143,12 +174,26 @@ trait TestReporter { result: &TestResult, elapsed: u64, ); + fn report_step_wait(&mut self, description: &TestStepDescription); + fn report_step_result( + &mut self, + description: &TestStepDescription, + result: &TestStepResult, + elapsed: u64, + ); fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration); } +enum DeferredStepOutput { + StepWait(TestStepDescription), + StepResult(TestStepDescription, TestStepResult, u64), +} + struct PrettyTestReporter { concurrent: bool, echo_output: bool, + deferred_step_output: HashMap>, + last_wait_output_level: usize, } impl PrettyTestReporter { @@ -156,6 +201,61 @@ impl PrettyTestReporter { PrettyTestReporter { concurrent, echo_output, + deferred_step_output: HashMap::new(), + last_wait_output_level: 0, + } + } + + fn force_report_wait(&mut self, description: &TestDescription) { + print!("test {} ...", description.name); + // flush for faster feedback when line buffered + std::io::stdout().flush().unwrap(); + self.last_wait_output_level = 0; + } + + fn force_report_step_wait(&mut self, description: &TestStepDescription) { + if self.last_wait_output_level < description.level { + println!(); + } + print!( + "{}test {} ...", + " ".repeat(description.level), + description.name + ); + // flush for faster feedback when line buffered + std::io::stdout().flush().unwrap(); + self.last_wait_output_level = description.level; + } + + fn force_report_step_result( + &mut self, + description: &TestStepDescription, + result: &TestStepResult, + elapsed: u64, + ) { + let status = match result { + TestStepResult::Ok => colors::green("ok").to_string(), + TestStepResult::Ignored => colors::yellow("ignored").to_string(), + TestStepResult::Pending(_) => colors::gray("pending").to_string(), + TestStepResult::Failed(_) => colors::red("FAILED").to_string(), + }; + + if self.last_wait_output_level == description.level { + print!(" "); + } else { + print!("{}", " ".repeat(description.level)); + } + + println!( + "{} {}", + status, + colors::gray(format!("({}ms)", elapsed)).to_string() + ); + + if let Some(error_text) = result.error() { + for line in error_text.lines() { + println!("{}{}", " ".repeat(description.level + 1), line); + } } } } @@ -168,7 +268,7 @@ impl TestReporter for PrettyTestReporter { fn report_wait(&mut self, description: &TestDescription) { if !self.concurrent { - print!("test {} ...", description.name); + self.force_report_wait(description); } } @@ -187,7 +287,27 @@ impl TestReporter for PrettyTestReporter { elapsed: u64, ) { if self.concurrent { - print!("test {} ...", description.name); + self.force_report_wait(description); + + if let Some(step_outputs) = self.deferred_step_output.remove(description) + { + for step_output in step_outputs { + match step_output { + DeferredStepOutput::StepWait(description) => { + self.force_report_step_wait(&description) + } + DeferredStepOutput::StepResult( + step_description, + step_result, + elapsed, + ) => self.force_report_step_result( + &step_description, + &step_result, + elapsed, + ), + } + } + } } let status = match result { @@ -196,13 +316,50 @@ impl TestReporter for PrettyTestReporter { TestResult::Failed(_) => colors::red("FAILED").to_string(), }; + if self.last_wait_output_level == 0 { + print!(" "); + } + println!( - " {} {}", + "{} {}", status, colors::gray(format!("({}ms)", elapsed)).to_string() ); } + fn report_step_wait(&mut self, description: &TestStepDescription) { + if self.concurrent { + self + .deferred_step_output + .entry(description.test.to_owned()) + .or_insert_with(Vec::new) + .push(DeferredStepOutput::StepWait(description.clone())); + } else { + self.force_report_step_wait(description); + } + } + + fn report_step_result( + &mut self, + description: &TestStepDescription, + result: &TestStepResult, + elapsed: u64, + ) { + if self.concurrent { + self + .deferred_step_output + .entry(description.test.to_owned()) + .or_insert_with(Vec::new) + .push(DeferredStepOutput::StepResult( + description.clone(), + result.clone(), + elapsed, + )); + } else { + self.force_report_step_result(description, result, elapsed); + } + } + fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration) { if !summary.failures.is_empty() { println!("\nfailures:\n"); @@ -650,11 +807,9 @@ async fn test_specifiers( TestResult::Ok => { summary.passed += 1; } - TestResult::Ignored => { summary.ignored += 1; } - TestResult::Failed(error) => { summary.failed += 1; summary.failures.push((description.clone(), error.clone())); @@ -663,6 +818,14 @@ async fn test_specifiers( reporter.report_result(&description, &result, elapsed); } + + TestEvent::StepWait(description) => { + reporter.report_step_wait(&description); + } + + TestEvent::StepResult(description, result, duration) => { + reporter.report_step_result(&description, &result, duration); + } } if let Some(x) = fail_fast { diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js index 92181cae12..2adb487fbe 100644 --- a/runtime/js/40_testing.js +++ b/runtime/js/40_testing.js @@ -11,7 +11,10 @@ const { ArrayPrototypeFilter, ArrayPrototypePush, + ArrayPrototypeSome, DateNow, + Error, + Function, JSONStringify, Promise, TypeError, @@ -21,7 +24,9 @@ StringPrototypeSlice, RegExp, RegExpPrototypeTest, + SymbolToStringTag, } = window.__bootstrap.primordials; + let testStepsEnabled = false; // Wrap test function in additional assertion that makes sure // the test case does not leak async "ops" - ie. number of async @@ -29,10 +34,10 @@ // ops. Note that "unref" ops are ignored since in nature that are // optional. function assertOps(fn) { - return async function asyncOpSanitizer() { + return async function asyncOpSanitizer(...params) { const pre = metrics(); try { - await fn(); + await fn(...params); } finally { // Defer until next event loop turn - that way timeouts and intervals // cleared can actually be removed from resource table, otherwise @@ -67,9 +72,9 @@ finishing test case.`, function assertResources( fn, ) { - return async function resourceSanitizer() { + return async function resourceSanitizer(...params) { const pre = core.resources(); - await fn(); + await fn(...params); const post = core.resources(); const preStr = JSONStringify(pre, null, 2); @@ -87,7 +92,7 @@ finishing test case.`; // Wrap test function in additional assertion that makes sure // that the test case does not accidentally exit prematurely. function assertExit(fn) { - return async function exitSanitizer() { + return async function exitSanitizer(...params) { setExitHandler((exitCode) => { assert( false, @@ -96,7 +101,7 @@ finishing test case.`; }); try { - await fn(); + await fn(...params); } catch (err) { throw err; } finally { @@ -105,6 +110,86 @@ finishing test case.`; }; } + function assertTestStepScopes(fn) { + /** @param step {TestStep} */ + return async function testStepSanitizer(step) { + preValidation(); + // only report waiting after pre-validation + if (step.canStreamReporting()) { + step.reportWait(); + } + await fn(createTestContext(step)); + postValidation(); + + function preValidation() { + const runningSteps = getPotentialConflictingRunningSteps(); + const runningStepsWithSanitizers = ArrayPrototypeFilter( + runningSteps, + (t) => t.usesSanitizer, + ); + + if (runningStepsWithSanitizers.length > 0) { + throw new Error( + "Cannot start test step while another test step with sanitizers is running.\n" + + runningStepsWithSanitizers + .map((s) => ` * ${s.getFullName()}`) + .join("\n"), + ); + } + + if (step.usesSanitizer && runningSteps.length > 0) { + throw new Error( + "Cannot start test step with sanitizers while another test step is running.\n" + + runningSteps.map((s) => ` * ${s.getFullName()}`).join("\n"), + ); + } + + function getPotentialConflictingRunningSteps() { + /** @type {TestStep[]} */ + const results = []; + + let childStep = step; + for (const ancestor of step.ancestors()) { + for (const siblingStep of ancestor.children) { + if (siblingStep === childStep) { + continue; + } + if (!siblingStep.finalized) { + ArrayPrototypePush(results, siblingStep); + } + } + childStep = ancestor; + } + return results; + } + } + + function postValidation() { + // check for any running steps + const hasRunningSteps = ArrayPrototypeSome( + step.children, + (r) => r.status === "pending", + ); + if (hasRunningSteps) { + throw new Error( + "There were still test steps running after the current scope finished execution. " + + "Ensure all steps are awaited (ex. `await t.step(...)`).", + ); + } + + // check if an ancestor already completed + for (const ancestor of step.ancestors()) { + if (ancestor.finalized) { + throw new Error( + "Parent scope completed before test step finished execution. " + + "Ensure all steps are awaited (ex. `await t.step(...)`).", + ); + } + } + } + }; + } + function withPermissions(fn, permissions) { function pledgePermissions(permissions) { return core.opSync( @@ -117,11 +202,11 @@ finishing test case.`; core.opSync("op_restore_test_permissions", token); } - return async function applyPermissions() { + return async function applyPermissions(...params) { const token = pledgePermissions(permissions); try { - await fn(); + await fn(...params); } finally { restorePermissions(token); } @@ -130,8 +215,7 @@ finishing test case.`; const tests = []; - // Main test function provided by Deno, as you can see it merely - // creates a new object with "name" and "fn" fields. + // Main test function provided by Deno. function test( t, fn, @@ -164,17 +248,7 @@ finishing test case.`; testDef = { ...defaults, ...t }; } - if (testDef.sanitizeOps) { - testDef.fn = assertOps(testDef.fn); - } - - if (testDef.sanitizeResources) { - testDef.fn = assertResources(testDef.fn); - } - - if (testDef.sanitizeExit) { - testDef.fn = assertExit(testDef.fn); - } + testDef.fn = wrapTestFnWithSanitizers(testDef.fn, testDef); if (testDef.permissions) { testDef.fn = withPermissions( @@ -186,7 +260,7 @@ finishing test case.`; ArrayPrototypePush(tests, testDef); } - function formatFailure(error) { + function formatError(error) { if (error.errors) { const message = error .errors @@ -195,12 +269,10 @@ finishing test case.`; ) .join("\n"); - return { - failed: error.name + "\n" + message + error.stack, - }; + return error.name + "\n" + message + error.stack; } - return { failed: inspectArgs([error]) }; + return inspectArgs([error]); } function createTestFilter(filter) { @@ -223,18 +295,40 @@ finishing test case.`; }; } - async function runTest({ ignore, fn }) { - if (ignore) { + async function runTest(test, description) { + if (test.ignore) { return "ignored"; } - try { - await fn(); - } catch (error) { - return formatFailure(error); - } + const step = new TestStep({ + name: test.name, + parent: undefined, + rootTestDescription: description, + sanitizeOps: test.sanitizeOps, + sanitizeResources: test.sanitizeResources, + sanitizeExit: test.sanitizeExit, + }); - return "ok"; + try { + await test.fn(step); + const failCount = step.failedChildStepsCount(); + return failCount === 0 ? "ok" : { + "failed": formatError( + new Error( + `${failCount} test step${failCount === 1 ? "" : "s"} failed.`, + ), + ), + }; + } catch (error) { + return { + "failed": formatError(error), + }; + } finally { + // ensure the children report their result + for (const child of step.children) { + child.reportResult(); + } + } } function getTestOrigin() { @@ -265,6 +359,18 @@ finishing test case.`; }); } + function reportTestStepWait(testDescription) { + core.opSync("op_dispatch_test_event", { + stepWait: testDescription, + }); + } + + function reportTestStepResult(testDescription, result, elapsed) { + core.opSync("op_dispatch_test_event", { + stepResult: [testDescription, result, elapsed], + }); + } + async function runTests({ filter = null, shuffle = null, @@ -314,7 +420,7 @@ finishing test case.`; reportTestWait(description); - const result = await runTest(test); + const result = await runTest(test, description); const elapsed = DateNow() - earlier; reportTestResult(description, result, elapsed); @@ -323,9 +429,341 @@ finishing test case.`; globalThis.console = originalConsole; } + /** + * @typedef {{ + * fn: (t: TestContext) => void | Promise, + * name: string, + * ignore?: boolean, + * sanitizeOps?: boolean, + * sanitizeResources?: boolean, + * sanitizeExit?: boolean, + * }} TestStepDefinition + * + * @typedef {{ + * name: string; + * parent: TestStep | undefined, + * rootTestDescription: { origin: string; name: string }; + * sanitizeOps: boolean, + * sanitizeResources: boolean, + * sanitizeExit: boolean, + * }} TestStepParams + */ + + class TestStep { + /** @type {TestStepParams} */ + #params; + reportedWait = false; + #reportedResult = false; + finalized = false; + elapsed = 0; + status = "pending"; + error = undefined; + /** @type {TestStep[]} */ + children = []; + + /** @param params {TestStepParams} */ + constructor(params) { + this.#params = params; + } + + get name() { + return this.#params.name; + } + + get parent() { + return this.#params.parent; + } + + get rootTestDescription() { + return this.#params.rootTestDescription; + } + + get sanitizerOptions() { + return { + sanitizeResources: this.#params.sanitizeResources, + sanitizeOps: this.#params.sanitizeOps, + sanitizeExit: this.#params.sanitizeExit, + }; + } + + get usesSanitizer() { + return this.#params.sanitizeResources || + this.#params.sanitizeOps || + this.#params.sanitizeExit; + } + + failedChildStepsCount() { + return ArrayPrototypeFilter( + this.children, + /** @param step {TestStep} */ + (step) => step.status === "failed", + ).length; + } + + canStreamReporting() { + // there should only ever be one sub step running when running with + // sanitizers, so we can use this to tell if we can stream reporting + return this.selfAndAllAncestorsUseSanitizer() && + this.children.every((c) => c.usesSanitizer || c.finalized); + } + + selfAndAllAncestorsUseSanitizer() { + if (!this.usesSanitizer) { + return false; + } + + for (const ancestor of this.ancestors()) { + if (!ancestor.usesSanitizer) { + return false; + } + } + + return true; + } + + *ancestors() { + let ancestor = this.parent; + while (ancestor) { + yield ancestor; + ancestor = ancestor.parent; + } + } + + getFullName() { + if (this.parent) { + return `${this.parent.getFullName()} > ${this.name}`; + } else { + return this.name; + } + } + + reportWait() { + if (this.reportedWait || !this.parent) { + return; + } + + reportTestStepWait(this.#getTestStepDescription()); + + this.reportedWait = true; + } + + reportResult() { + if (this.#reportedResult || !this.parent) { + return; + } + + this.reportWait(); + + for (const child of this.children) { + child.reportResult(); + } + + reportTestStepResult( + this.#getTestStepDescription(), + this.#getStepResult(), + this.elapsed, + ); + + this.#reportedResult = true; + } + + #getStepResult() { + switch (this.status) { + case "ok": + return "ok"; + case "ignored": + return "ignored"; + case "pending": + return { + "pending": this.error && formatError(this.error), + }; + case "failed": + return { + "failed": this.error && formatError(this.error), + }; + default: + throw new Error(`Unhandled status: ${this.status}`); + } + } + + #getTestStepDescription() { + return { + test: this.rootTestDescription, + name: this.name, + level: this.#getLevel(), + }; + } + + #getLevel() { + let count = 0; + for (const _ of this.ancestors()) { + count++; + } + return count; + } + } + + /** @param parentStep {TestStep} */ + function createTestContext(parentStep) { + return { + [SymbolToStringTag]: "TestContext", + /** + * @param nameOrTestDefinition {string | TestStepDefinition} + * @param fn {(t: TestContext) => void | Promise} + */ + async step(nameOrTestDefinition, fn) { + if (!testStepsEnabled) { + throw new Error( + "Test steps are unstable. The --unstable flag must be provided.", + ); + } + + if (parentStep.finalized) { + throw new Error( + "Cannot run test step after parent scope has finished execution. " + + "Ensure any `.step(...)` calls are executed before their parent scope completes execution.", + ); + } + + const definition = getDefinition(); + const subStep = new TestStep({ + name: definition.name, + parent: parentStep, + rootTestDescription: parentStep.rootTestDescription, + sanitizeOps: getOrDefault( + definition.sanitizeOps, + parentStep.sanitizerOptions.sanitizeOps, + ), + sanitizeResources: getOrDefault( + definition.sanitizeResources, + parentStep.sanitizerOptions.sanitizeResources, + ), + sanitizeExit: getOrDefault( + definition.sanitizeExit, + parentStep.sanitizerOptions.sanitizeExit, + ), + }); + + ArrayPrototypePush(parentStep.children, subStep); + + try { + if (definition.ignore) { + subStep.status = "ignored"; + subStep.finalized = true; + if (subStep.canStreamReporting()) { + subStep.reportResult(); + } + return false; + } + + const testFn = wrapTestFnWithSanitizers( + definition.fn, + subStep.sanitizerOptions, + ); + const start = DateNow(); + + try { + await testFn(subStep); + + if (subStep.failedChildStepsCount() > 0) { + subStep.status = "failed"; + } else { + subStep.status = "ok"; + } + } catch (error) { + subStep.error = formatError(error); + subStep.status = "failed"; + } + + subStep.elapsed = DateNow() - start; + + if (subStep.parent?.finalized) { + // always point this test out as one that was still running + // if the parent step finalized + subStep.status = "pending"; + } + + subStep.finalized = true; + + if (subStep.reportedWait && subStep.canStreamReporting()) { + subStep.reportResult(); + } + + return subStep.status === "ok"; + } finally { + if (parentStep.canStreamReporting()) { + // flush any buffered steps + for (const parentChild of parentStep.children) { + parentChild.reportResult(); + } + } + } + + /** @returns {TestStepDefinition} */ + function getDefinition() { + if (typeof nameOrTestDefinition === "string") { + if (!(fn instanceof Function)) { + throw new TypeError("Expected function for second argument."); + } + return { + name: nameOrTestDefinition, + fn, + }; + } else if (typeof nameOrTestDefinition === "object") { + return nameOrTestDefinition; + } else { + throw new TypeError( + "Expected a test definition or name and function.", + ); + } + } + }, + }; + } + + /** + * @template T {Function} + * @param testFn {T} + * @param opts {{ + * sanitizeOps: boolean, + * sanitizeResources: boolean, + * sanitizeExit: boolean, + * }} + * @returns {T} + */ + function wrapTestFnWithSanitizers(testFn, opts) { + testFn = assertTestStepScopes(testFn); + + if (opts.sanitizeOps) { + testFn = assertOps(testFn); + } + if (opts.sanitizeResources) { + testFn = assertResources(testFn); + } + if (opts.sanitizeExit) { + testFn = assertExit(testFn); + } + return testFn; + } + + /** + * @template T + * @param value {T | undefined} + * @param defaultValue {T} + * @returns T + */ + function getOrDefault(value, defaultValue) { + return value == null ? defaultValue : value; + } + + function enableTestSteps() { + testStepsEnabled = true; + } + window.__bootstrap.internals = { ...window.__bootstrap.internals ?? {}, runTests, + enableTestSteps, }; window.__bootstrap.testing = { diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 117200a287..32732923be 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -213,6 +213,9 @@ delete Object.prototype.__proto__; runtimeOptions.v8Version, runtimeOptions.tsVersion, ); + if (runtimeOptions.unstableFlag) { + internals.enableTestSteps(); + } build.setBuildInfo(runtimeOptions.target); util.setLogDebug(runtimeOptions.debugFlag, source); const prepareStackTrace = core.createPrepareStackTrace(