feat: Add support for "deno test --fail-fast=N" (#11316)

This commit adds support for specifying threshold in the "--fail-fast"
flag for "deno test" subcommand. Previously using "--fail-fast" stopped
running the test suite after first failure and with this change users
may specify number of failed tests that will cause the suite to be interrupted.

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

cli/flags.rs

@@ -98,7 +98,7 @@ pub enum DenoSubcommand {
   Test {
     doc: bool,
     no_run: bool,
-    fail_fast: bool,
+    fail_fast: Option<usize>,
     quiet: bool,
     allow_none: bool,
     include: Option<Vec<String>>,
@@ -1001,8 +1001,23 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
       Arg::with_name("fail-fast")
         .long("fail-fast")
         .alias("failfast")
-        .help("Stop on first error")
-        .takes_value(false),
+        .help("Stop after N errors. Defaults to stopping after first failure.")
+        .min_values(0)
+        .required(false)
+        .takes_value(true)
+        .require_equals(true)
+        .value_name("N")
+        .validator(|val: String| match val.parse::<usize>() {
+          Ok(val) => {
+            if val == 0 {
+              return Err(
+                "fail-fast should be an number greater than 0".to_string(),
+              );
+            }
+            Ok(())
+          }
+          Err(_) => Err("fail-fast should be a number".to_string()),
+        }),
     )
     .arg(
       Arg::with_name("allow-none")
@@ -1696,11 +1711,20 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
 
   let no_run = matches.is_present("no-run");
   let doc = matches.is_present("doc");
-  let fail_fast = matches.is_present("fail-fast");
   let allow_none = matches.is_present("allow-none");
   let quiet = matches.is_present("quiet");
   let filter = matches.value_of("filter").map(String::from);
 
+  let fail_fast = if matches.is_present("fail-fast") {
+    if let Some(value) = matches.value_of("fail-fast") {
+      Some(value.parse().unwrap())
+    } else {
+      Some(1)
+    }
+  } else {
+    None
+  };
+
   let shuffle = if matches.is_present("shuffle") {
     let value = if let Some(value) = matches.value_of("shuffle") {
       value.parse::<u64>().unwrap()
@@ -3387,7 +3411,7 @@ mod tests {
         subcommand: DenoSubcommand::Test {
           no_run: true,
           doc: false,
-          fail_fast: false,
+          fail_fast: None,
           filter: Some("- foo".to_string()),
           allow_none: true,
           quiet: false,
@@ -3426,6 +3450,29 @@ mod tests {
     );
   }
 
+  #[test]
+  fn test_with_fail_fast() {
+    let r = flags_from_vec(svec!["deno", "test", "--fail-fast=3"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Test {
+          no_run: false,
+          doc: false,
+          fail_fast: Some(3),
+          filter: None,
+          allow_none: false,
+          quiet: false,
+          shuffle: None,
+          include: None,
+          concurrent_jobs: 1,
+          terse: false
+        },
+        ..Flags::default()
+      }
+    );
+  }
+
   #[test]
   fn bundle_with_cafile() {
     let r = flags_from_vec(svec![

cli/main.rs

@@ -987,7 +987,7 @@ async fn test_command(
   include: Option<Vec<String>>,
   no_run: bool,
   doc: bool,
-  fail_fast: bool,
+  fail_fast: Option<usize>,
   quiet: bool,
   allow_none: bool,
   filter: Option<String>,

cli/tests/test/fail_fast_with_val.out

@@ -0,0 +1,23 @@
+[WILDCARD]
+running 10 tests from [WILDCARD]/test/fail_fast_with_val.ts
+test test 1 ... FAILED ([WILDCARD])
+test test 2 ... FAILED ([WILDCARD])
+
+failures:
+
+test 1
+Error
+    at [WILDCARD]/test/fail_fast_with_val.ts:2:9
+    at [WILDCARD]
+
+test 2
+Error
+    at [WILDCARD]/test/fail_fast_with_val.ts:5:9
+    at [WILDCARD]
+
+failures:
+
+	test 1
+	test 2
+
+test result: FAILED. 0 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

cli/tests/test/fail_fast_with_val.ts

@@ -0,0 +1,30 @@
+Deno.test("test 1", () => {
+  throw new Error();
+});
+Deno.test("test 2", () => {
+  throw new Error();
+});
+Deno.test("test 3", () => {
+  throw new Error();
+});
+Deno.test("test 4", () => {
+  throw new Error();
+});
+Deno.test("test 5", () => {
+  throw new Error();
+});
+Deno.test("test 6", () => {
+  throw new Error();
+});
+Deno.test("test 7", () => {
+  throw new Error();
+});
+Deno.test("test 8", () => {
+  throw new Error();
+});
+Deno.test("test 9", () => {
+  throw new Error();
+});
+Deno.test("test 0", () => {
+  throw new Error();
+});

cli/tools/test_runner.rs

@@ -340,7 +340,7 @@ pub async fn run_tests(
   doc_modules: Vec<ModuleSpecifier>,
   test_modules: Vec<ModuleSpecifier>,
   no_run: bool,
-  fail_fast: bool,
+  fail_fast: Option<usize>,
   quiet: bool,
   allow_none: bool,
   filter: Option<String>,
@@ -515,6 +515,7 @@ pub async fn run_tests(
       let mut has_error = false;
       let mut planned = 0;
       let mut reported = 0;
+      let mut failed = 0;
 
       for event in receiver.iter() {
         match event.message.clone() {
@@ -538,6 +539,7 @@ pub async fn run_tests(
 
             if let TestResult::Failed(_) = result {
               has_error = true;
+              failed += 1;
             }
           }
           _ => {}
@@ -545,8 +547,10 @@ pub async fn run_tests(
 
         reporter.visit_event(event);
 
-        if has_error && fail_fast {
-          break;
+        if let Some(x) = fail_fast {
+          if failed >= x {
+            break;
+          }
         }
       }