feat(unstable/test): imperative test steps API (#12190)

cli/dts/lib.deno.ns.d.ts

@@ -113,8 +113,12 @@ declare namespace Deno {
    * See: https://no-color.org/ */
   export const noColor: boolean;
 
+  /** **UNSTABLE**: New option, yet to be vetted. */
+  export interface TestContext {
+  }
+
   export interface TestDefinition {
-    fn: () => void | Promise<void>;
+    fn: (t: TestContext) => void | Promise<void>;
     name: string;
     ignore?: boolean;
     /** If at least one test has `only` set to true, only run tests that have
@@ -127,7 +131,6 @@ declare namespace Deno {
      * after the test has exactly the same contents as before the test. Defaults
      * to true. */
     sanitizeResources?: boolean;
-
     /** Ensure the test case does not prematurely cause the process to exit,
      * for example via a call to `Deno.exit`. Defaults to true. */
     sanitizeExit?: boolean;
@@ -184,7 +187,10 @@ declare namespace Deno {
    * });
    * ```
    */
-  export function test(name: string, fn: () => void | Promise<void>): void;
+  export function test(
+    name: string,
+    fn: (t: TestContext) => void | Promise<void>,
+  ): void;
 
   /** Exit the Deno process with optional exit code. If no exit code is supplied
    * then Deno will exit with return code of 0.

cli/dts/lib.deno.unstable.d.ts

@@ -948,6 +948,43 @@ declare namespace Deno {
     };
   }
 
+  /** **UNSTABLE**: New option, yet to be vetted. */
+  export interface TestContext {
+    /** Run a sub step of the parent test with a given name. Returns a promise
+     * that resolves to a boolean signifying if the step completed successfully.
+     * The returned promise never rejects unless the arguments are invalid.
+     * If the test was ignored, the promise returns `false`.
+     */
+    step(t: TestStepDefinition): Promise<boolean>;
+
+    /** Run a sub step of the parent test with a given name. Returns a promise
+     * that resolves to a boolean signifying if the step completed successfully.
+     * The returned promise never rejects unless the arguments are invalid.
+     * If the test was ignored, the promise returns `false`.
+     */
+    step(
+      name: string,
+      fn: (t: TestContext) => void | Promise<void>,
+    ): Promise<boolean>;
+  }
+
+  /** **UNSTABLE**: New option, yet to be vetted. */
+  export interface TestStepDefinition {
+    fn: (t: TestContext) => void | Promise<void>;
+    name: string;
+    ignore?: boolean;
+    /** Check that the number of async completed ops after the test is the same
+     * as number of dispatched ops. Defaults to true. */
+    sanitizeOps?: boolean;
+    /** Ensure the test case does not "leak" resources - ie. the resource table
+     * after the test has exactly the same contents as before the test. Defaults
+     * to true. */
+    sanitizeResources?: boolean;
+    /** Ensure the test case does not prematurely cause the process to exit,
+     * for example via a call to `Deno.exit`. Defaults to true. */
+    sanitizeExit?: boolean;
+  }
+
   /** **UNSTABLE**: new API, yet to be vetted.
    *
    * A generic transport listener for message-oriented protocols. */

cli/tests/integration/test_tests.rs

@@ -186,3 +186,39 @@ itest!(aggregate_error {
   exit_code: 1,
   output: "test/aggregate_error.out",
 });
+
+itest!(steps_passing_steps {
+  args: "test --unstable test/steps/passing_steps.ts",
+  exit_code: 0,
+  output: "test/steps/passing_steps.out",
+});
+
+itest!(steps_passing_steps_concurrent {
+  args: "test --unstable --jobs=2 test/steps/passing_steps.ts",
+  exit_code: 0,
+  output: "test/steps/passing_steps.out",
+});
+
+itest!(steps_failing_steps {
+  args: "test --unstable test/steps/failing_steps.ts",
+  exit_code: 1,
+  output: "test/steps/failing_steps.out",
+});
+
+itest!(steps_ignored_steps {
+  args: "test --unstable test/steps/ignored_steps.ts",
+  exit_code: 0,
+  output: "test/steps/ignored_steps.out",
+});
+
+itest!(steps_invalid_usage {
+  args: "test --unstable test/steps/invalid_usage.ts",
+  exit_code: 1,
+  output: "test/steps/invalid_usage.out",
+});
+
+itest!(steps_no_unstable_flag {
+  args: "test test/steps/no_unstable_flag.ts",
+  exit_code: 1,
+  output: "test/steps/no_unstable_flag.out",
+});

cli/tests/testdata/test/steps/failing_steps.out

@@ -0,0 +1,53 @@
+[WILDCARD]
+running 3 tests from [WILDCARD]/failing_steps.ts
+test nested failure ...
+  test step 1 ...
+    test inner 1 ... FAILED ([WILDCARD])
+      Error: Failed.
+          at [WILDCARD]/failing_steps.ts:[WILDCARD]
+          [WILDCARD]
+    test inner 2 ... ok ([WILDCARD])
+  FAILED ([WILDCARD])
+FAILED ([WILDCARD])
+test multiple test step failures ...
+  test step 1 ... FAILED ([WILDCARD])
+    Error: Fail.
+        [WILDCARD]
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Fail.
+        at [WILDCARD]/failing_steps.ts:[WILDCARD]
+        [WILDCARD]
+FAILED ([WILDCARD])
+test failing step in failing test ...
+  test step 1 ... FAILED ([WILDCARD])
+    Error: Fail.
+        at [WILDCARD]/failing_steps.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+
+failures:
+
+nested failure
+Error: 1 test step failed.
+    at runTest (deno:runtime/js/40_testing.js:[WILDCARD])
+    at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD])
+
+multiple test step failures
+Error: 2 test steps failed.
+    at runTest (deno:runtime/js/40_testing.js:[WILDCARD])
+    at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD])
+
+failing step in failing test
+Error: Fail test.
+    at [WILDCARD]/failing_steps.ts:[WILDCARD]
+    at [WILDCARD]
+
+failures:
+
+	nested failure
+	multiple test step failures
+	failing step in failing test
+
+test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
+error: Test failed

cli/tests/testdata/test/steps/failing_steps.ts

@@ -0,0 +1,27 @@
+Deno.test("nested failure", async (t) => {
+  const success = await t.step("step 1", async (t) => {
+    let success = await t.step("inner 1", () => {
+      throw new Error("Failed.");
+    });
+    if (success) throw new Error("Expected failure");
+
+    success = await t.step("inner 2", () => {});
+    if (!success) throw new Error("Expected success");
+  });
+
+  if (success) throw new Error("Expected failure");
+});
+
+Deno.test("multiple test step failures", async (t) => {
+  await t.step("step 1", () => {
+    throw new Error("Fail.");
+  });
+  await t.step("step 2", () => Promise.reject(new Error("Fail.")));
+});
+
+Deno.test("failing step in failing test", async (t) => {
+  await t.step("step 1", () => {
+    throw new Error("Fail.");
+  });
+  throw new Error("Fail test.");
+});

cli/tests/testdata/test/steps/ignored_steps.out

@@ -0,0 +1,8 @@
+[WILDCARD]
+running 1 test from [WILDCARD]/ignored_steps.ts
+test ignored step ...
+  test step 1 ... ignored ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]

cli/tests/testdata/test/steps/ignored_steps.ts

@@ -0,0 +1,16 @@
+Deno.test("ignored step", async (t) => {
+  let result = await t.step({
+    name: "step 1",
+    ignore: true,
+    fn: () => {
+      throw new Error("Fail.");
+    },
+  });
+  if (result !== false) throw new Error("Expected false.");
+  result = await t.step({
+    name: "step 2",
+    ignore: false,
+    fn: () => {},
+  });
+  if (result !== true) throw new Error("Expected true.");
+});

cli/tests/testdata/test/steps/invalid_usage.out

@@ -0,0 +1,111 @@
+[WILDCARD]
+running 7 tests from [WILDCARD]/invalid_usage.ts
+test capturing ...
+  test some step ... ok ([WILDCARD])
+FAILED ([WILDCARD])
+test top level missing await ...
+  test step ... pending ([WILDCARD])
+FAILED ([WILDCARD])
+test inner missing await ...
+  test step ...
+    test inner ... pending ([WILDCARD])
+      Error: Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+          at postValidation [WILDCARD]
+          at testStepSanitizer [WILDCARD]
+  FAILED ([WILDCARD])
+    Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+        at postValidation [WILDCARD]
+        at testStepSanitizer [WILDCARD]
+        at async fn ([WILDCARD]/invalid_usage.ts:[WILDCARD])
+        at async Object.testStepSanitizer [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps with sanitizers ...
+  test step 1 ... pending ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step while another test step with sanitizers is running.
+     * parallel steps with sanitizers > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps when first has sanitizer ...
+  test step 1 ... pending ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step while another test step with sanitizers is running.
+     * parallel steps when first has sanitizer > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps when second has sanitizer ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step with sanitizers while another test step is running.
+     * parallel steps when second has sanitizer > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps where only inner tests have sanitizers ...
+  test step 1 ...
+    test step inner ... ok ([WILDCARD])
+  ok ([WILDCARD])
+  test step 2 ...
+    test step inner ... FAILED ([WILDCARD])
+      Error: Cannot start test step with sanitizers while another test step is running.
+       * parallel steps where only inner tests have sanitizers > step 1
+          at preValidation ([WILDCARD])
+          at testStepSanitizer ([WILDCARD])
+          at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+  FAILED ([WILDCARD])
+FAILED ([WILDCARD])
+
+failures:
+
+capturing
+Error: Cannot run test step after parent scope has finished execution. Ensure any `.step(...)` calls are executed before their parent scope completes execution.
+    at TestContext.step ([WILDCARD])
+    at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+    at [WILDCARD]
+
+top level missing await
+Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+    at postValidation [WILDCARD]
+    at testStepSanitizer ([WILDCARD])
+    [WILDCARD]
+
+inner missing await
+Error: 1 test step failed.
+    at [WILDCARD]
+
+parallel steps with sanitizers
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+parallel steps when first has sanitizer
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+parallel steps when second has sanitizer
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+failures:
+
+	capturing
+	top level missing await
+	inner missing await
+	parallel steps with sanitizers
+	parallel steps when first has sanitizer
+	parallel steps when second has sanitizer
+	parallel steps where only inner tests have sanitizers
+
+test result: FAILED. 0 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
+error: Test failed

cli/tests/testdata/test/steps/invalid_usage.ts

@@ -0,0 +1,122 @@
+import { deferred } from "../../../../../test_util/std/async/deferred.ts";
+
+Deno.test("capturing", async (t) => {
+  let capturedContext!: Deno.TestContext;
+  await t.step("some step", (t) => {
+    capturedContext = t;
+  });
+  // this should error because the scope of the tester has already completed
+  await capturedContext.step("next step", () => {});
+});
+
+Deno.test("top level missing await", (t) => {
+  t.step("step", () => {
+    return new Promise((resolve) => setTimeout(resolve, 10));
+  });
+});
+
+Deno.test({
+  name: "inner missing await",
+  fn: async (t) => {
+    await t.step("step", (t) => {
+      t.step("inner", () => {
+        return new Promise((resolve) => setTimeout(resolve, 10));
+      });
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test("parallel steps with sanitizers", async (t) => {
+  // not allowed because steps with sanitizers cannot be run in parallel
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step("step 1", async () => {
+    step1Entered.resolve();
+    await step2Finished;
+  });
+  await step1Entered;
+  await t.step("step 2", () => {});
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test("parallel steps when first has sanitizer", async (t) => {
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step({
+    name: "step 1",
+    fn: async () => {
+      step1Entered.resolve();
+      await step2Finished;
+    },
+  });
+  await step1Entered;
+  await t.step({
+    name: "step 2",
+    fn: () => {},
+    sanitizeOps: false,
+    sanitizeResources: false,
+    sanitizeExit: false,
+  });
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test("parallel steps when second has sanitizer", async (t) => {
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step({
+    name: "step 1",
+    fn: async () => {
+      step1Entered.resolve();
+      await step2Finished;
+    },
+    sanitizeOps: false,
+    sanitizeResources: false,
+    sanitizeExit: false,
+  });
+  await step1Entered;
+  await t.step({
+    name: "step 2",
+    fn: async () => {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    },
+  });
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test({
+  name: "parallel steps where only inner tests have sanitizers",
+  fn: async (t) => {
+    const step1Entered = deferred();
+    const step2Finished = deferred();
+    const step1 = t.step("step 1", async (t) => {
+      await t.step({
+        name: "step inner",
+        fn: async () => {
+          step1Entered.resolve();
+          await step2Finished;
+        },
+        sanitizeOps: true,
+      });
+    });
+    await step1Entered;
+    await t.step("step 2", async (t) => {
+      await t.step({
+        name: "step inner",
+        fn: () => {},
+        sanitizeOps: true,
+      });
+    });
+    step2Finished.resolve();
+    await step1;
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});

cli/tests/testdata/test/steps/no_unstable_flag.out

@@ -0,0 +1,13 @@
+[WILDCARD]
+running 1 test from [WILDCARD]/no_unstable_flag.ts
+test description ... FAILED ([WILDCARD])
+
+failures:
+
+description
+Error: Test steps are unstable. The --unstable flag must be provided.
+    at [WILDCARD]
+
+failures:
+
+[WILDCARD]

cli/tests/testdata/test/steps/no_unstable_flag.ts

@@ -0,0 +1,4 @@
+Deno.test("description", async (t) => {
+  // deno-lint-ignore no-explicit-any
+  await (t as any).step("step", () => {});
+});

cli/tests/testdata/test/steps/passing_steps.out

@@ -0,0 +1,38 @@
+[WILDCARD]
+running 5 tests from [WILDCARD]
+test description ...
+  test step 1 ...
+    test inner 1 ... ok ([WILDCARD]ms)
+    test inner 2 ... ok ([WILDCARD]ms)
+  ok ([WILDCARD]ms)
+ok ([WILDCARD]ms)
+test parallel steps without sanitizers ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+test parallel steps without sanitizers due to parent ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+test steps with disabled sanitizers, then enabled, then parallel disabled ...
+  test step 1 ...
+    test step 1 ...
+      test step 1 ...
+        test step 1 ... ok ([WILDCARD])
+        test step 1 ... ok ([WILDCARD])
+      ok ([WILDCARD])
+      test step 2 ... ok ([WILDCARD])
+    ok ([WILDCARD])
+  ok ([WILDCARD])
+ok ([WILDCARD])
+test steps buffered then streaming reporting ...
+  test step 1 ...
+    test step 1 - 1 ... ok ([WILDCARD])
+    test step 1 - 2 ...
+      test step 1 - 2 - 1 ... ok ([WILDCARD])
+    ok ([WILDCARD])
+  ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+
+test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]

cli/tests/testdata/test/steps/passing_steps.ts

@@ -0,0 +1,120 @@
+import { deferred } from "../../../../../test_util/std/async/deferred.ts";
+
+Deno.test("description", async (t) => {
+  const success = await t.step("step 1", async (t) => {
+    await t.step("inner 1", () => {});
+    await t.step("inner 2", () => {});
+  });
+
+  if (!success) throw new Error("Expected the step to return true.");
+});
+
+Deno.test("parallel steps without sanitizers", async (t) => {
+  // allowed
+  await Promise.all([
+    t.step({
+      name: "step 1",
+      fn: async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      },
+      sanitizeOps: false,
+      sanitizeResources: false,
+      sanitizeExit: false,
+    }),
+    t.step({
+      name: "step 2",
+      fn: async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      },
+      sanitizeOps: false,
+      sanitizeResources: false,
+      sanitizeExit: false,
+    }),
+  ]);
+});
+
+Deno.test({
+  name: "parallel steps without sanitizers due to parent",
+  fn: async (t) => {
+    // allowed because parent disabled the sanitizers
+    await Promise.all([
+      t.step("step 1", async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }),
+      t.step("step 2", async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }),
+    ]);
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test({
+  name: "steps with disabled sanitizers, then enabled, then parallel disabled",
+  fn: async (t) => {
+    await t.step("step 1", async (t) => {
+      await t.step({
+        name: "step 1",
+        fn: async (t) => {
+          await Promise.all([
+            t.step({
+              name: "step 1",
+              fn: async (t) => {
+                await new Promise((resolve) => setTimeout(resolve, 10));
+                await Promise.all([
+                  t.step("step 1", () => {}),
+                  t.step("step 1", () => {}),
+                ]);
+              },
+              sanitizeExit: false,
+              sanitizeResources: false,
+              sanitizeOps: false,
+            }),
+            t.step({
+              name: "step 2",
+              fn: () => {},
+              sanitizeResources: false,
+              sanitizeOps: false,
+              sanitizeExit: false,
+            }),
+          ]);
+        },
+        sanitizeResources: true,
+        sanitizeOps: true,
+        sanitizeExit: true,
+      });
+    });
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test("steps buffered then streaming reporting", async (t) => {
+  // no sanitizers so this will be buffered
+  await t.step({
+    name: "step 1",
+    fn: async (t) => {
+      // also ensure the buffered tests display in order regardless of the second one finishing first
+      const step2Finished = deferred();
+      const step1 = t.step("step 1 - 1", async () => {
+        await step2Finished;
+      });
+      const step2 = t.step("step 1 - 2", async (t) => {
+        await t.step("step 1 - 2 - 1", () => {});
+      });
+      await step2;
+      step2Finished.resolve();
+      await step1;
+    },
+    sanitizeResources: false,
+    sanitizeOps: false,
+    sanitizeExit: false,
+  });
+
+  // now this will start streaming and we want to
+  // ensure it flushes the buffer of the last test
+  await t.step("step 2", async () => {});
+});

cli/tests/unit/test_util.ts

@@ -39,7 +39,7 @@ interface UnitTestOptions {
   permissions?: UnitTestPermissions;
 }
 
-type TestFunction = () => void | Promise<void>;
+type TestFunction = (tester: Deno.TestContext) => void | Promise<void>;
 
 export function unitTest(fn: TestFunction): void;
 export function unitTest(options: UnitTestOptions, fn: TestFunction): void;

cli/tests/unit/testing_test.ts

@@ -1,5 +1,5 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
-import { assertThrows, unitTest } from "./test_util.ts";
+import { assertRejects, assertThrows, unitTest } from "./test_util.ts";
 
 unitTest(function testFnOverloading() {
   // just verifying that you can use this test definition syntax
@@ -25,3 +25,41 @@ unitTest(function nameOfTestCaseCantBeEmpty() {
     "The test name can't be empty",
   );
 });
+
+unitTest(function invalidStepArguments(t) {
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step("test");
+    },
+    TypeError,
+    "Expected function for second argument.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step("test", "not a function");
+    },
+    TypeError,
+    "Expected function for second argument.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step();
+    },
+    TypeError,
+    "Expected a test definition or name and function.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step(() => {});
+    },
+    TypeError,
+    "Expected a test definition or name and function.",
+  );
+});

cli/tools/test.rs

@@ -39,7 +39,9 @@ use rand::seq::SliceRandom;
 use rand::SeedableRng;
 use regex::Regex;
 use serde::Deserialize;
+use std::collections::HashMap;
 use std::collections::HashSet;
+use std::io::Write;
 use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::sync::mpsc::channel;
@@ -60,7 +62,7 @@ enum TestMode {
   Both,
 }
 
-#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub struct TestDescription {
   pub origin: String,
@@ -82,6 +84,33 @@ pub enum TestResult {
   Failed(String),
 }
 
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct TestStepDescription {
+  pub test: TestDescription,
+  pub level: usize,
+  pub name: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum TestStepResult {
+  Ok,
+  Ignored,
+  Failed(Option<String>),
+  Pending(Option<String>),
+}
+
+impl TestStepResult {
+  fn error(&self) -> Option<&str> {
+    match self {
+      TestStepResult::Failed(Some(text)) => Some(text.as_str()),
+      TestStepResult::Pending(Some(text)) => Some(text.as_str()),
+      _ => None,
+    }
+  }
+}
+
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct TestPlan {
@@ -98,6 +127,8 @@ pub enum TestEvent {
   Wait(TestDescription),
   Output(TestOutput),
   Result(TestDescription, TestResult, u64),
+  StepWait(TestStepDescription),
+  StepResult(TestStepDescription, TestStepResult, u64),
 }
 
 #[derive(Debug, Clone, Deserialize)]
@@ -143,12 +174,26 @@ trait TestReporter {
     result: &TestResult,
     elapsed: u64,
   );
+  fn report_step_wait(&mut self, description: &TestStepDescription);
+  fn report_step_result(
+    &mut self,
+    description: &TestStepDescription,
+    result: &TestStepResult,
+    elapsed: u64,
+  );
   fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration);
 }
 
+enum DeferredStepOutput {
+  StepWait(TestStepDescription),
+  StepResult(TestStepDescription, TestStepResult, u64),
+}
+
 struct PrettyTestReporter {
   concurrent: bool,
   echo_output: bool,
+  deferred_step_output: HashMap<TestDescription, Vec<DeferredStepOutput>>,
+  last_wait_output_level: usize,
 }
 
 impl PrettyTestReporter {
@@ -156,6 +201,61 @@ impl PrettyTestReporter {
     PrettyTestReporter {
       concurrent,
       echo_output,
+      deferred_step_output: HashMap::new(),
+      last_wait_output_level: 0,
+    }
+  }
+
+  fn force_report_wait(&mut self, description: &TestDescription) {
+    print!("test {} ...", description.name);
+    // flush for faster feedback when line buffered
+    std::io::stdout().flush().unwrap();
+    self.last_wait_output_level = 0;
+  }
+
+  fn force_report_step_wait(&mut self, description: &TestStepDescription) {
+    if self.last_wait_output_level < description.level {
+      println!();
+    }
+    print!(
+      "{}test {} ...",
+      "  ".repeat(description.level),
+      description.name
+    );
+    // flush for faster feedback when line buffered
+    std::io::stdout().flush().unwrap();
+    self.last_wait_output_level = description.level;
+  }
+
+  fn force_report_step_result(
+    &mut self,
+    description: &TestStepDescription,
+    result: &TestStepResult,
+    elapsed: u64,
+  ) {
+    let status = match result {
+      TestStepResult::Ok => colors::green("ok").to_string(),
+      TestStepResult::Ignored => colors::yellow("ignored").to_string(),
+      TestStepResult::Pending(_) => colors::gray("pending").to_string(),
+      TestStepResult::Failed(_) => colors::red("FAILED").to_string(),
+    };
+
+    if self.last_wait_output_level == description.level {
+      print!(" ");
+    } else {
+      print!("{}", "  ".repeat(description.level));
+    }
+
+    println!(
+      "{} {}",
+      status,
+      colors::gray(format!("({}ms)", elapsed)).to_string()
+    );
+
+    if let Some(error_text) = result.error() {
+      for line in error_text.lines() {
+        println!("{}{}", "  ".repeat(description.level + 1), line);
+      }
     }
   }
 }
@@ -168,7 +268,7 @@ impl TestReporter for PrettyTestReporter {
 
   fn report_wait(&mut self, description: &TestDescription) {
     if !self.concurrent {
-      print!("test {} ...", description.name);
+      self.force_report_wait(description);
     }
   }
 
@@ -187,7 +287,27 @@ impl TestReporter for PrettyTestReporter {
     elapsed: u64,
   ) {
     if self.concurrent {
-      print!("test {} ...", description.name);
+      self.force_report_wait(description);
+
+      if let Some(step_outputs) = self.deferred_step_output.remove(description)
+      {
+        for step_output in step_outputs {
+          match step_output {
+            DeferredStepOutput::StepWait(description) => {
+              self.force_report_step_wait(&description)
+            }
+            DeferredStepOutput::StepResult(
+              step_description,
+              step_result,
+              elapsed,
+            ) => self.force_report_step_result(
+              &step_description,
+              &step_result,
+              elapsed,
+            ),
+          }
+        }
+      }
     }
 
     let status = match result {
@@ -196,13 +316,50 @@ impl TestReporter for PrettyTestReporter {
       TestResult::Failed(_) => colors::red("FAILED").to_string(),
     };
 
+    if self.last_wait_output_level == 0 {
+      print!(" ");
+    }
+
     println!(
-      " {} {}",
+      "{} {}",
       status,
       colors::gray(format!("({}ms)", elapsed)).to_string()
     );
   }
 
+  fn report_step_wait(&mut self, description: &TestStepDescription) {
+    if self.concurrent {
+      self
+        .deferred_step_output
+        .entry(description.test.to_owned())
+        .or_insert_with(Vec::new)
+        .push(DeferredStepOutput::StepWait(description.clone()));
+    } else {
+      self.force_report_step_wait(description);
+    }
+  }
+
+  fn report_step_result(
+    &mut self,
+    description: &TestStepDescription,
+    result: &TestStepResult,
+    elapsed: u64,
+  ) {
+    if self.concurrent {
+      self
+        .deferred_step_output
+        .entry(description.test.to_owned())
+        .or_insert_with(Vec::new)
+        .push(DeferredStepOutput::StepResult(
+          description.clone(),
+          result.clone(),
+          elapsed,
+        ));
+    } else {
+      self.force_report_step_result(description, result, elapsed);
+    }
+  }
+
   fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration) {
     if !summary.failures.is_empty() {
       println!("\nfailures:\n");
@@ -650,11 +807,9 @@ async fn test_specifiers(
               TestResult::Ok => {
                 summary.passed += 1;
               }
-
               TestResult::Ignored => {
                 summary.ignored += 1;
               }
-
               TestResult::Failed(error) => {
                 summary.failed += 1;
                 summary.failures.push((description.clone(), error.clone()));
@@ -663,6 +818,14 @@ async fn test_specifiers(
 
             reporter.report_result(&description, &result, elapsed);
           }
+
+          TestEvent::StepWait(description) => {
+            reporter.report_step_wait(&description);
+          }
+
+          TestEvent::StepResult(description, result, duration) => {
+            reporter.report_step_result(&description, &result, duration);
+          }
         }
 
         if let Some(x) = fail_fast {

runtime/js/40_testing.js

@@ -11,7 +11,10 @@
   const {
     ArrayPrototypeFilter,
     ArrayPrototypePush,
+    ArrayPrototypeSome,
     DateNow,
+    Error,
+    Function,
     JSONStringify,
     Promise,
     TypeError,
@@ -21,7 +24,9 @@
     StringPrototypeSlice,
     RegExp,
     RegExpPrototypeTest,
+    SymbolToStringTag,
   } = window.__bootstrap.primordials;
+  let testStepsEnabled = false;
 
   // Wrap test function in additional assertion that makes sure
   // the test case does not leak async "ops" - ie. number of async
@@ -29,10 +34,10 @@
   // ops. Note that "unref" ops are ignored since in nature that are
   // optional.
   function assertOps(fn) {
-    return async function asyncOpSanitizer() {
+    return async function asyncOpSanitizer(...params) {
       const pre = metrics();
       try {
-        await fn();
+        await fn(...params);
       } finally {
         // Defer until next event loop turn - that way timeouts and intervals
         // cleared can actually be removed from resource table, otherwise
@@ -67,9 +72,9 @@ finishing test case.`,
   function assertResources(
     fn,
   ) {
-    return async function resourceSanitizer() {
+    return async function resourceSanitizer(...params) {
       const pre = core.resources();
-      await fn();
+      await fn(...params);
       const post = core.resources();
 
       const preStr = JSONStringify(pre, null, 2);
@@ -87,7 +92,7 @@ finishing test case.`;
   // Wrap test function in additional assertion that makes sure
   // that the test case does not accidentally exit prematurely.
   function assertExit(fn) {
-    return async function exitSanitizer() {
+    return async function exitSanitizer(...params) {
       setExitHandler((exitCode) => {
         assert(
           false,
@@ -96,7 +101,7 @@ finishing test case.`;
       });
 
       try {
-        await fn();
+        await fn(...params);
       } catch (err) {
         throw err;
       } finally {
@@ -105,6 +110,86 @@ finishing test case.`;
     };
   }
 
+  function assertTestStepScopes(fn) {
+    /** @param step {TestStep} */
+    return async function testStepSanitizer(step) {
+      preValidation();
+      // only report waiting after pre-validation
+      if (step.canStreamReporting()) {
+        step.reportWait();
+      }
+      await fn(createTestContext(step));
+      postValidation();
+
+      function preValidation() {
+        const runningSteps = getPotentialConflictingRunningSteps();
+        const runningStepsWithSanitizers = ArrayPrototypeFilter(
+          runningSteps,
+          (t) => t.usesSanitizer,
+        );
+
+        if (runningStepsWithSanitizers.length > 0) {
+          throw new Error(
+            "Cannot start test step while another test step with sanitizers is running.\n" +
+              runningStepsWithSanitizers
+                .map((s) => ` * ${s.getFullName()}`)
+                .join("\n"),
+          );
+        }
+
+        if (step.usesSanitizer && runningSteps.length > 0) {
+          throw new Error(
+            "Cannot start test step with sanitizers while another test step is running.\n" +
+              runningSteps.map((s) => ` * ${s.getFullName()}`).join("\n"),
+          );
+        }
+
+        function getPotentialConflictingRunningSteps() {
+          /** @type {TestStep[]} */
+          const results = [];
+
+          let childStep = step;
+          for (const ancestor of step.ancestors()) {
+            for (const siblingStep of ancestor.children) {
+              if (siblingStep === childStep) {
+                continue;
+              }
+              if (!siblingStep.finalized) {
+                ArrayPrototypePush(results, siblingStep);
+              }
+            }
+            childStep = ancestor;
+          }
+          return results;
+        }
+      }
+
+      function postValidation() {
+        // check for any running steps
+        const hasRunningSteps = ArrayPrototypeSome(
+          step.children,
+          (r) => r.status === "pending",
+        );
+        if (hasRunningSteps) {
+          throw new Error(
+            "There were still test steps running after the current scope finished execution. " +
+              "Ensure all steps are awaited (ex. `await t.step(...)`).",
+          );
+        }
+
+        // check if an ancestor already completed
+        for (const ancestor of step.ancestors()) {
+          if (ancestor.finalized) {
+            throw new Error(
+              "Parent scope completed before test step finished execution. " +
+                "Ensure all steps are awaited (ex. `await t.step(...)`).",
+            );
+          }
+        }
+      }
+    };
+  }
+
   function withPermissions(fn, permissions) {
     function pledgePermissions(permissions) {
       return core.opSync(
@@ -117,11 +202,11 @@ finishing test case.`;
       core.opSync("op_restore_test_permissions", token);
     }
 
-    return async function applyPermissions() {
+    return async function applyPermissions(...params) {
       const token = pledgePermissions(permissions);
 
       try {
-        await fn();
+        await fn(...params);
       } finally {
         restorePermissions(token);
       }
@@ -130,8 +215,7 @@ finishing test case.`;
 
   const tests = [];
 
-  // Main test function provided by Deno, as you can see it merely
-  // creates a new object with "name" and "fn" fields.
+  // Main test function provided by Deno.
   function test(
     t,
     fn,
@@ -164,17 +248,7 @@ finishing test case.`;
       testDef = { ...defaults, ...t };
     }
 
-    if (testDef.sanitizeOps) {
-      testDef.fn = assertOps(testDef.fn);
-    }
-
-    if (testDef.sanitizeResources) {
-      testDef.fn = assertResources(testDef.fn);
-    }
-
-    if (testDef.sanitizeExit) {
-      testDef.fn = assertExit(testDef.fn);
-    }
+    testDef.fn = wrapTestFnWithSanitizers(testDef.fn, testDef);
 
     if (testDef.permissions) {
       testDef.fn = withPermissions(
@@ -186,7 +260,7 @@ finishing test case.`;
     ArrayPrototypePush(tests, testDef);
   }
 
-  function formatFailure(error) {
+  function formatError(error) {
     if (error.errors) {
       const message = error
         .errors
@@ -195,12 +269,10 @@ finishing test case.`;
         )
         .join("\n");
 
-      return {
-        failed: error.name + "\n" + message + error.stack,
-      };
+      return error.name + "\n" + message + error.stack;
     }
 
-    return { failed: inspectArgs([error]) };
+    return inspectArgs([error]);
   }
 
   function createTestFilter(filter) {
@@ -223,18 +295,40 @@ finishing test case.`;
     };
   }
 
-  async function runTest({ ignore, fn }) {
-    if (ignore) {
+  async function runTest(test, description) {
+    if (test.ignore) {
       return "ignored";
     }
 
-    try {
-      await fn();
-    } catch (error) {
-      return formatFailure(error);
-    }
+    const step = new TestStep({
+      name: test.name,
+      parent: undefined,
+      rootTestDescription: description,
+      sanitizeOps: test.sanitizeOps,
+      sanitizeResources: test.sanitizeResources,
+      sanitizeExit: test.sanitizeExit,
+    });
 
-    return "ok";
+    try {
+      await test.fn(step);
+      const failCount = step.failedChildStepsCount();
+      return failCount === 0 ? "ok" : {
+        "failed": formatError(
+          new Error(
+            `${failCount} test step${failCount === 1 ? "" : "s"} failed.`,
+          ),
+        ),
+      };
+    } catch (error) {
+      return {
+        "failed": formatError(error),
+      };
+    } finally {
+      // ensure the children report their result
+      for (const child of step.children) {
+        child.reportResult();
+      }
+    }
   }
 
   function getTestOrigin() {
@@ -265,6 +359,18 @@ finishing test case.`;
     });
   }
 
+  function reportTestStepWait(testDescription) {
+    core.opSync("op_dispatch_test_event", {
+      stepWait: testDescription,
+    });
+  }
+
+  function reportTestStepResult(testDescription, result, elapsed) {
+    core.opSync("op_dispatch_test_event", {
+      stepResult: [testDescription, result, elapsed],
+    });
+  }
+
   async function runTests({
     filter = null,
     shuffle = null,
@@ -314,7 +420,7 @@ finishing test case.`;
 
       reportTestWait(description);
 
-      const result = await runTest(test);
+      const result = await runTest(test, description);
       const elapsed = DateNow() - earlier;
 
       reportTestResult(description, result, elapsed);
@@ -323,9 +429,341 @@ finishing test case.`;
     globalThis.console = originalConsole;
   }
 
+  /**
+   * @typedef {{
+   *   fn: (t: TestContext) => void | Promise<void>,
+   *   name: string,
+   *   ignore?: boolean,
+   *   sanitizeOps?: boolean,
+   *   sanitizeResources?: boolean,
+   *   sanitizeExit?: boolean,
+   * }} TestStepDefinition
+   *
+   * @typedef {{
+   *   name: string;
+   *   parent: TestStep | undefined,
+   *   rootTestDescription: { origin: string; name: string };
+   *   sanitizeOps: boolean,
+   *   sanitizeResources: boolean,
+   *   sanitizeExit: boolean,
+   * }} TestStepParams
+   */
+
+  class TestStep {
+    /** @type {TestStepParams} */
+    #params;
+    reportedWait = false;
+    #reportedResult = false;
+    finalized = false;
+    elapsed = 0;
+    status = "pending";
+    error = undefined;
+    /** @type {TestStep[]} */
+    children = [];
+
+    /** @param params {TestStepParams} */
+    constructor(params) {
+      this.#params = params;
+    }
+
+    get name() {
+      return this.#params.name;
+    }
+
+    get parent() {
+      return this.#params.parent;
+    }
+
+    get rootTestDescription() {
+      return this.#params.rootTestDescription;
+    }
+
+    get sanitizerOptions() {
+      return {
+        sanitizeResources: this.#params.sanitizeResources,
+        sanitizeOps: this.#params.sanitizeOps,
+        sanitizeExit: this.#params.sanitizeExit,
+      };
+    }
+
+    get usesSanitizer() {
+      return this.#params.sanitizeResources ||
+        this.#params.sanitizeOps ||
+        this.#params.sanitizeExit;
+    }
+
+    failedChildStepsCount() {
+      return ArrayPrototypeFilter(
+        this.children,
+        /** @param step {TestStep} */
+        (step) => step.status === "failed",
+      ).length;
+    }
+
+    canStreamReporting() {
+      // there should only ever be one sub step running when running with
+      // sanitizers, so we can use this to tell if we can stream reporting
+      return this.selfAndAllAncestorsUseSanitizer() &&
+        this.children.every((c) => c.usesSanitizer || c.finalized);
+    }
+
+    selfAndAllAncestorsUseSanitizer() {
+      if (!this.usesSanitizer) {
+        return false;
+      }
+
+      for (const ancestor of this.ancestors()) {
+        if (!ancestor.usesSanitizer) {
+          return false;
+        }
+      }
+
+      return true;
+    }
+
+    *ancestors() {
+      let ancestor = this.parent;
+      while (ancestor) {
+        yield ancestor;
+        ancestor = ancestor.parent;
+      }
+    }
+
+    getFullName() {
+      if (this.parent) {
+        return `${this.parent.getFullName()} > ${this.name}`;
+      } else {
+        return this.name;
+      }
+    }
+
+    reportWait() {
+      if (this.reportedWait || !this.parent) {
+        return;
+      }
+
+      reportTestStepWait(this.#getTestStepDescription());
+
+      this.reportedWait = true;
+    }
+
+    reportResult() {
+      if (this.#reportedResult || !this.parent) {
+        return;
+      }
+
+      this.reportWait();
+
+      for (const child of this.children) {
+        child.reportResult();
+      }
+
+      reportTestStepResult(
+        this.#getTestStepDescription(),
+        this.#getStepResult(),
+        this.elapsed,
+      );
+
+      this.#reportedResult = true;
+    }
+
+    #getStepResult() {
+      switch (this.status) {
+        case "ok":
+          return "ok";
+        case "ignored":
+          return "ignored";
+        case "pending":
+          return {
+            "pending": this.error && formatError(this.error),
+          };
+        case "failed":
+          return {
+            "failed": this.error && formatError(this.error),
+          };
+        default:
+          throw new Error(`Unhandled status: ${this.status}`);
+      }
+    }
+
+    #getTestStepDescription() {
+      return {
+        test: this.rootTestDescription,
+        name: this.name,
+        level: this.#getLevel(),
+      };
+    }
+
+    #getLevel() {
+      let count = 0;
+      for (const _ of this.ancestors()) {
+        count++;
+      }
+      return count;
+    }
+  }
+
+  /** @param parentStep {TestStep} */
+  function createTestContext(parentStep) {
+    return {
+      [SymbolToStringTag]: "TestContext",
+      /**
+       * @param nameOrTestDefinition {string | TestStepDefinition}
+       * @param fn {(t: TestContext) => void | Promise<void>}
+       */
+      async step(nameOrTestDefinition, fn) {
+        if (!testStepsEnabled) {
+          throw new Error(
+            "Test steps are unstable. The --unstable flag must be provided.",
+          );
+        }
+
+        if (parentStep.finalized) {
+          throw new Error(
+            "Cannot run test step after parent scope has finished execution. " +
+              "Ensure any `.step(...)` calls are executed before their parent scope completes execution.",
+          );
+        }
+
+        const definition = getDefinition();
+        const subStep = new TestStep({
+          name: definition.name,
+          parent: parentStep,
+          rootTestDescription: parentStep.rootTestDescription,
+          sanitizeOps: getOrDefault(
+            definition.sanitizeOps,
+            parentStep.sanitizerOptions.sanitizeOps,
+          ),
+          sanitizeResources: getOrDefault(
+            definition.sanitizeResources,
+            parentStep.sanitizerOptions.sanitizeResources,
+          ),
+          sanitizeExit: getOrDefault(
+            definition.sanitizeExit,
+            parentStep.sanitizerOptions.sanitizeExit,
+          ),
+        });
+
+        ArrayPrototypePush(parentStep.children, subStep);
+
+        try {
+          if (definition.ignore) {
+            subStep.status = "ignored";
+            subStep.finalized = true;
+            if (subStep.canStreamReporting()) {
+              subStep.reportResult();
+            }
+            return false;
+          }
+
+          const testFn = wrapTestFnWithSanitizers(
+            definition.fn,
+            subStep.sanitizerOptions,
+          );
+          const start = DateNow();
+
+          try {
+            await testFn(subStep);
+
+            if (subStep.failedChildStepsCount() > 0) {
+              subStep.status = "failed";
+            } else {
+              subStep.status = "ok";
+            }
+          } catch (error) {
+            subStep.error = formatError(error);
+            subStep.status = "failed";
+          }
+
+          subStep.elapsed = DateNow() - start;
+
+          if (subStep.parent?.finalized) {
+            // always point this test out as one that was still running
+            // if the parent step finalized
+            subStep.status = "pending";
+          }
+
+          subStep.finalized = true;
+
+          if (subStep.reportedWait && subStep.canStreamReporting()) {
+            subStep.reportResult();
+          }
+
+          return subStep.status === "ok";
+        } finally {
+          if (parentStep.canStreamReporting()) {
+            // flush any buffered steps
+            for (const parentChild of parentStep.children) {
+              parentChild.reportResult();
+            }
+          }
+        }
+
+        /** @returns {TestStepDefinition} */
+        function getDefinition() {
+          if (typeof nameOrTestDefinition === "string") {
+            if (!(fn instanceof Function)) {
+              throw new TypeError("Expected function for second argument.");
+            }
+            return {
+              name: nameOrTestDefinition,
+              fn,
+            };
+          } else if (typeof nameOrTestDefinition === "object") {
+            return nameOrTestDefinition;
+          } else {
+            throw new TypeError(
+              "Expected a test definition or name and function.",
+            );
+          }
+        }
+      },
+    };
+  }
+
+  /**
+   * @template T {Function}
+   * @param testFn {T}
+   * @param opts {{
+   *   sanitizeOps: boolean,
+   *   sanitizeResources: boolean,
+   *   sanitizeExit: boolean,
+   * }}
+   * @returns {T}
+   */
+  function wrapTestFnWithSanitizers(testFn, opts) {
+    testFn = assertTestStepScopes(testFn);
+
+    if (opts.sanitizeOps) {
+      testFn = assertOps(testFn);
+    }
+    if (opts.sanitizeResources) {
+      testFn = assertResources(testFn);
+    }
+    if (opts.sanitizeExit) {
+      testFn = assertExit(testFn);
+    }
+    return testFn;
+  }
+
+  /**
+   * @template T
+   * @param value {T | undefined}
+   * @param defaultValue {T}
+   * @returns T
+   */
+  function getOrDefault(value, defaultValue) {
+    return value == null ? defaultValue : value;
+  }
+
+  function enableTestSteps() {
+    testStepsEnabled = true;
+  }
+
   window.__bootstrap.internals = {
     ...window.__bootstrap.internals ?? {},
     runTests,
+    enableTestSteps,
   };
 
   window.__bootstrap.testing = {

runtime/js/99_main.js

@@ -213,6 +213,9 @@ delete Object.prototype.__proto__;
       runtimeOptions.v8Version,
       runtimeOptions.tsVersion,
     );
+    if (runtimeOptions.unstableFlag) {
+      internals.enableTestSteps();
+    }
     build.setBuildInfo(runtimeOptions.target);
     util.setLogDebug(runtimeOptions.debugFlag, source);
     const prepareStackTrace = core.createPrepareStackTrace(