feat: add exit sanitizer to Deno.test (#9529)

This adds an exit sanitizer to ensure that code being tested or 
dependencies of that code can't accidentally call "Deno.exit"
leading to partial test runs and false results.

cli/dts/lib.deno.ns.d.ts

@@ -108,6 +108,10 @@ declare namespace Deno {
      * after the test has exactly the same contents as before the test. Defaults
      * to true. */
     sanitizeResources?: boolean;
+
+    /** Ensure the test case does not prematurely cause the process to exit,
+     * for example via a call to `Deno.exit`. Defaults to true. */
+    sanitizeExit?: boolean;
   }
 
   /** Register a test which will be run when `deno test` is used on the command

cli/tests/exit_sanitizer_test.out

@@ -0,0 +1,28 @@
+Check [WILDCARD]/$deno$test.ts
+running 3 tests
+test exit(0) ... FAILED ([WILDCARD])
+test exit(1) ... FAILED ([WILDCARD])
+test exit(2) ... FAILED ([WILDCARD])
+
+failures:
+
+exit(0)
+AssertionError: Test case attempted to exit with exit code: 0
+    [WILDCARD]
+
+exit(1)
+AssertionError: Test case attempted to exit with exit code: 1
+    [WILDCARD]
+
+exit(2)
+AssertionError: Test case attempted to exit with exit code: 2
+    [WILDCARD]
+
+failures:
+
+	exit(0)
+	exit(1)
+	exit(2)
+
+test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+

cli/tests/exit_sanitizer_test.ts

@@ -0,0 +1,11 @@
+Deno.test("exit(0)", function () {
+  Deno.exit(0);
+});
+
+Deno.test("exit(1)", function () {
+  Deno.exit(1);
+});
+
+Deno.test("exit(2)", function () {
+  Deno.exit(2);
+});

cli/tests/integration_tests.rs

@@ -2252,6 +2252,12 @@ mod integration {
     assert!(out.contains("test result: FAILED. 1 passed; 1 failed; 1 ignored; 0 measured; 0 filtered out"));
   }
 
+  itest!(test_exit_sanitizer {
+    args: "test exit_sanitizer_test.ts",
+    output: "exit_sanitizer_test.out",
+    exit_code: 1,
+  });
+
   itest!(stdout_write_all {
     args: "run --quiet stdout_write_all.ts",
     output: "stdout_write_all.out",

cli/tests/test_finally_cleartimeout.out

@@ -7,13 +7,7 @@ failures:
 
 error
 Error: fail
-    at [WILDCARD]/test_finally_cleartimeout.ts:4:11
-    at asyncOpSanitizer (deno:runtime/js/40_testing.js:38:15)
-    at Object.resourceSanitizer [as fn] (deno:runtime/js/40_testing.js:74:13)
-    at TestRunner.[Symbol.asyncIterator] (deno:runtime/js/40_testing.js:249:24)
-    at AsyncGenerator.next (<anonymous>)
-    at Object.runTests (deno:runtime/js/40_testing.js:326:22)
-    at async [WILDCARD]/$deno$test.ts:3:1
+    [WILDCARD]
 
 failures:
 

docs/testing.md

@@ -92,6 +92,31 @@ Deno.test({
 });
 ```
 
+### Exit sanitizer
+
+There's also the exit sanitizer which ensures that tested code doesn't call
+Deno.exit() signaling a false test success.
+
+This is enabled by default for all tests, but can be disabled by setting the
+`sanitizeExit` boolean to false in thetest definition.
+
+```ts
+Deno.test({
+  name: "false success",
+  fn() {
+    Deno.exit(0);
+  },
+  sanitizeExit: false,
+});
+
+Deno.test({
+  name: "failing test",
+  fn() {
+    throw new Error("this test fails");
+  },
+});
+```
+
 ## Running tests
 
 To run the test, call `deno test` with the file that contains your test

runtime/js/30_os.js

@@ -24,6 +24,13 @@
     return core.jsonOpSync("op_system_cpu_info");
   }
 
+  // This is an internal only method used by the test harness to override the
+  // behavior of exit when the exit sanitizer is enabled.
+  let exitHandler = null;
+  function setExitHandler(fn) {
+    exitHandler = fn;
+  }
+
   function exit(code = 0) {
     // Dispatches `unload` only when it's not dispatched yet.
     if (!window[Symbol.for("isUnloadDispatched")]) {
@@ -31,6 +38,12 @@
       // ref: https://github.com/denoland/deno/issues/3603
       window.dispatchEvent(new Event("unload"));
     }
+
+    if (exitHandler) {
+      exitHandler(code);
+      return;
+    }
+
     core.jsonOpSync("op_exit", { code });
     throw new Error("Code not reachable");
   }
@@ -63,6 +76,7 @@
   window.__bootstrap.os = {
     env,
     execPath,
+    setExitHandler,
     exit,
     osRelease,
     systemMemoryInfo,

runtime/js/40_testing.js

@@ -4,7 +4,7 @@
 ((window) => {
   const core = window.Deno.core;
   const colors = window.__bootstrap.colors;
-  const { exit } = window.__bootstrap.os;
+  const { setExitHandler, exit } = window.__bootstrap.os;
   const { Console, inspectArgs } = window.__bootstrap.console;
   const { stdout } = window.__bootstrap.files;
   const { exposeForTest } = window.__bootstrap.internals;
@@ -86,6 +86,27 @@ finishing test case.`;
     };
   }
 
+  // Wrap test function in additional assertion that makes sure
+  // that the test case does not accidentally exit prematurely.
+  function assertExit(fn) {
+    return async function exitSanitizer() {
+      setExitHandler((exitCode) => {
+        assert(
+          false,
+          `Test case attempted to exit with exit code: ${exitCode}`,
+        );
+      });
+
+      try {
+        await fn();
+      } catch (err) {
+        throw err;
+      } finally {
+        setExitHandler(null);
+      }
+    };
+  }
+
   const TEST_REGISTRY = [];
 
   // Main test function provided by Deno, as you can see it merely
@@ -100,6 +121,7 @@ finishing test case.`;
       only: false,
       sanitizeOps: true,
       sanitizeResources: true,
+      sanitizeExit: true,
     };
 
     if (typeof t === "string") {
@@ -128,6 +150,10 @@ finishing test case.`;
       testDef.fn = assertResources(testDef.fn);
     }
 
+    if (testDef.sanitizeExit) {
+      testDef.fn = assertExit(testDef.fn);
+    }
+
     TEST_REGISTRY.push(testDef);
   }