refactor(test): support custom writer in PrettyTestReporter (#20783)

cli/lsp/testing/execution.rs

@@ -404,6 +404,7 @@ impl TestRun {
                 );
               }
             }
+            test::TestEvent::ForceEndReport => {}
             test::TestEvent::Sigint => {}
           }
         }

cli/tools/test/fmt.rs

@@ -3,7 +3,9 @@
 use super::*;
 
 pub fn to_relative_path_or_remote_url(cwd: &Url, path_or_url: &str) -> String {
-  let url = Url::parse(path_or_url).unwrap();
+  let Ok(url) = Url::parse(path_or_url) else {
+    return "<anonymous>".to_string();
+  };
   if url.scheme() == "file" {
     if let Some(mut r) = cwd.make_relative(&url) {
       if !r.starts_with("../") {

cli/tools/test/mod.rs

@@ -50,6 +50,7 @@ use deno_runtime::fmt_errors::format_js_error;
 use deno_runtime::permissions::Permissions;
 use deno_runtime::permissions::PermissionsContainer;
 use deno_runtime::tokio_util::create_and_run_current_thread;
+use deno_runtime::worker::MainWorker;
 use indexmap::IndexMap;
 use indexmap::IndexSet;
 use log::Level;
@@ -77,11 +78,12 @@ use std::time::Instant;
 use std::time::SystemTime;
 use tokio::signal;
 use tokio::sync::mpsc::unbounded_channel;
+use tokio::sync::mpsc::UnboundedReceiver;
 use tokio::sync::mpsc::UnboundedSender;
 use tokio::sync::mpsc::WeakUnboundedSender;
 
 pub mod fmt;
-mod reporters;
+pub mod reporters;
 
 pub use fmt::format_test_error;
 use reporters::CompoundTestReporter;
@@ -313,6 +315,7 @@ pub enum TestEvent {
   StepRegister(TestStepDescription),
   StepWait(usize),
   StepResult(usize, TestStepResult, u64),
+  ForceEndReport,
   Sigint,
 }
 
@@ -342,7 +345,7 @@ struct TestSpecifiersOptions {
   junit_path: Option<String>,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Default, Clone)]
 pub struct TestSpecifierOptions {
   pub shuffle: Option<u64>,
   pub filter: TestFilter,
@@ -379,6 +382,7 @@ fn get_test_reporter(options: &TestSpecifiersOptions) -> Box<dyn TestReporter> {
       parallel,
       options.log_level != Some(Level::Error),
       options.filter,
+      false,
     )),
     TestReporterConfig::Junit => {
       Box::new(JunitTestReporter::new("-".to_string()))
@@ -453,10 +457,35 @@ pub async fn test_specifier(
 
   worker.dispatch_load_event(located_script_name!())?;
 
-  let tests = {
+  run_tests_for_worker(&mut worker, &specifier, &options, &fail_fast_tracker)
+    .await?;
+
+  // Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
+  // event loop to continue beyond what's needed to await results.
+  worker.dispatch_beforeunload_event(located_script_name!())?;
+  worker.dispatch_unload_event(located_script_name!())?;
+
+  if let Some(coverage_collector) = coverage_collector.as_mut() {
+    worker
+      .with_event_loop(coverage_collector.stop_collecting().boxed_local())
+      .await?;
+  }
+  Ok(())
+}
+
+pub async fn run_tests_for_worker(
+  worker: &mut MainWorker,
+  specifier: &ModuleSpecifier,
+  options: &TestSpecifierOptions,
+  fail_fast_tracker: &FailFastTracker,
+) -> Result<(), AnyError> {
+  let (tests, mut sender) = {
     let state_rc = worker.js_runtime.op_state();
     let mut state = state_rc.borrow_mut();
-    std::mem::take(&mut state.borrow_mut::<ops::testing::TestContainer>().0)
+    (
+      std::mem::take(&mut state.borrow_mut::<ops::testing::TestContainer>().0),
+      state.borrow::<TestEventSender>().clone(),
+    )
   };
   let unfiltered = tests.len();
   let tests = tests
@@ -532,17 +561,6 @@ pub async fn test_specifier(
     let elapsed = SystemTime::now().duration_since(earlier)?.as_millis();
     sender.send(TestEvent::Result(desc.id, result, elapsed as u64))?;
   }
-
-  // Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
-  // event loop to continue beyond what's needed to await results.
-  worker.dispatch_beforeunload_event(located_script_name!())?;
-  worker.dispatch_unload_event(located_script_name!())?;
-
-  if let Some(coverage_collector) = coverage_collector.as_mut() {
-    worker
-      .with_event_loop(coverage_collector.stop_collecting().boxed_local())
-      .await?;
-  }
   Ok(())
 }
 
@@ -810,7 +828,7 @@ async fn test_specifiers(
     specifiers
   };
 
-  let (sender, mut receiver) = unbounded_channel::<TestEvent>();
+  let (sender, receiver) = unbounded_channel::<TestEvent>();
   let sender = TestEventSender::new(sender);
   let concurrent_jobs = options.concurrent_jobs;
 
@@ -820,7 +838,7 @@ async fn test_specifiers(
     sender_.upgrade().map(|s| s.send(TestEvent::Sigint).ok());
   });
   HAS_TEST_RUN_SIGINT_HANDLER.store(true, Ordering::Relaxed);
-  let mut reporter = get_test_reporter(&options);
+  let reporter = get_test_reporter(&options);
   let fail_fast_tracker = FailFastTracker::new(options.fail_fast);
 
   let join_handles = specifiers.into_iter().map(move |specifier| {
@@ -840,18 +858,34 @@ async fn test_specifiers(
       ))
     })
   });
-
   let join_stream = stream::iter(join_handles)
     .buffer_unordered(concurrent_jobs.get())
     .collect::<Vec<Result<Result<(), AnyError>, tokio::task::JoinError>>>();
 
-  let handler = {
+  let handler = spawn(async move { report_tests(receiver, reporter).await.0 });
-    spawn(async move {
+
-      let earlier = Instant::now();
+  let (join_results, result) = future::join(join_stream, handler).await;
+  sigint_handler_handle.abort();
+  HAS_TEST_RUN_SIGINT_HANDLER.store(false, Ordering::Relaxed);
+  for join_result in join_results {
+    join_result??;
+  }
+  result??;
+
+  Ok(())
+}
+
+/// Gives receiver back in case it was ended with `TestEvent::ForceEndReport`.
+pub async fn report_tests(
+  mut receiver: UnboundedReceiver<TestEvent>,
+  mut reporter: Box<dyn TestReporter>,
+) -> (Result<(), AnyError>, UnboundedReceiver<TestEvent>) {
   let mut tests = IndexMap::new();
   let mut test_steps = IndexMap::new();
   let mut tests_started = HashSet::new();
   let mut tests_with_result = HashSet::new();
+  let mut start_time = None;
+  let mut had_plan = false;
   let mut used_only = false;
   let mut failed = false;
 
@@ -861,25 +895,24 @@ async fn test_specifiers(
         reporter.report_register(&description);
         tests.insert(description.id, description);
       }
-
       TestEvent::Plan(plan) => {
+        if !had_plan {
+          start_time = Some(Instant::now());
+          had_plan = true;
+        }
         if plan.used_only {
           used_only = true;
         }
-
         reporter.report_plan(&plan);
       }
-
       TestEvent::Wait(id) => {
         if tests_started.insert(id) {
           reporter.report_wait(tests.get(&id).unwrap());
         }
       }
-
       TestEvent::Output(output) => {
         reporter.report_output(&output);
       }
-
       TestEvent::Result(id, result, elapsed) => {
         if tests_with_result.insert(id) {
           match result {
@@ -891,23 +924,19 @@ async fn test_specifiers(
           reporter.report_result(tests.get(&id).unwrap(), &result, elapsed);
         }
       }
-
       TestEvent::UncaughtError(origin, error) => {
         failed = true;
         reporter.report_uncaught_error(&origin, error);
       }
-
       TestEvent::StepRegister(description) => {
         reporter.report_step_register(&description);
         test_steps.insert(description.id, description);
       }
-
       TestEvent::StepWait(id) => {
         if tests_started.insert(id) {
           reporter.report_step_wait(test_steps.get(&id).unwrap());
         }
       }
-
       TestEvent::StepResult(id, result, duration) => {
         if tests_with_result.insert(id) {
           reporter.report_step_result(
@@ -919,9 +948,13 @@ async fn test_specifiers(
           );
         }
       }
-
+      TestEvent::ForceEndReport => {
+        break;
+      }
       TestEvent::Sigint => {
-            let elapsed = Instant::now().duration_since(earlier);
+        let elapsed = start_time
+          .map(|t| Instant::now().duration_since(t))
+          .unwrap_or_default();
         reporter.report_sigint(
           &tests_started
             .difference(&tests_with_result)
@@ -930,9 +963,7 @@ async fn test_specifiers(
           &tests,
           &test_steps,
         );
-            if let Err(err) =
+        if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) {
-              reporter.flush_report(&elapsed, &tests, &test_steps)
-            {
           eprint!("Test reporter failed to flush: {}", err)
         }
         std::process::exit(130);
@@ -940,42 +971,34 @@ async fn test_specifiers(
     }
   }
 
-      sigint_handler_handle.abort();
+  let elapsed = start_time
-      HAS_TEST_RUN_SIGINT_HANDLER.store(false, Ordering::Relaxed);
+    .map(|t| Instant::now().duration_since(t))
-
+    .unwrap_or_default();
-      let elapsed = Instant::now().duration_since(earlier);
   reporter.report_summary(&elapsed, &tests, &test_steps);
   if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) {
-        return Err(generic_error(format!(
+    return (
+      Err(generic_error(format!(
         "Test reporter failed to flush: {}",
         err
-        )));
+      ))),
+      receiver,
+    );
   }
 
   if used_only {
-        return Err(generic_error(
+    return (
+      Err(generic_error(
         "Test failed because the \"only\" option was used",
-        ));
+      )),
+      receiver,
+    );
   }
 
   if failed {
-        return Err(generic_error("Test failed"));
+    return (Err(generic_error("Test failed")), receiver);
   }
 
-      Ok(())
+  (Ok(()), receiver)
-    })
-  };
-
-  let (join_results, result) = future::join(join_stream, handler).await;
-
-  // propagate any errors
-  for join_result in join_results {
-    join_result??;
-  }
-
-  result??;
-
-  Ok(())
 }
 
 /// Checks if the path has a basename and extension Deno supports for tests.
@@ -1300,7 +1323,7 @@ pub async fn run_tests_with_watch(
 
 /// Tracks failures for the `--fail-fast` argument in
 /// order to tell when to stop running tests.
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct FailFastTracker {
   max_count: Option<usize>,
   failure_count: Arc<AtomicUsize>,

cli/tools/test/reporters/common.rs

@@ -66,6 +66,7 @@ pub fn format_test_step_for_summary(
 }
 
 pub(super) fn report_sigint(
+  writer: &mut dyn std::io::Write,
   cwd: &Url,
   tests_pending: &HashSet<usize>,
   tests: &IndexMap<usize, TestDescription>,
@@ -84,17 +85,20 @@ pub(super) fn report_sigint(
         .insert(format_test_step_for_summary(cwd, desc, tests, test_steps));
     }
   }
-  println!(
+  writeln!(
+    writer,
     "\n{} The following tests were pending:\n",
     colors::intense_blue("SIGINT")
-  );
+  )
+  .unwrap();
   for entry in formatted_pending {
-    println!("{}", entry);
+    writeln!(writer, "{}", entry).unwrap();
   }
-  println!();
+  writeln!(writer).unwrap();
 }
 
 pub(super) fn report_summary(
+  writer: &mut dyn std::io::Write,
   cwd: &Url,
   summary: &TestSummary,
   elapsed: &Duration,
@@ -120,14 +124,20 @@ pub(super) fn report_summary(
     }
 
     // note: the trailing whitespace is intentional to get a red background
-    println!("\n{}\n", colors::white_bold_on_red(" ERRORS "));
+    writeln!(writer, "\n{}\n", colors::white_bold_on_red(" ERRORS ")).unwrap();
     for (origin, (failures, uncaught_error)) in failures_by_origin {
       for (description, failure) in failures {
         if !failure.hide_in_summary() {
           let failure_title = format_test_for_summary(cwd, description);
-          println!("{}", &failure_title);
+          writeln!(writer, "{}", &failure_title).unwrap();
-          println!("{}: {}", colors::red_bold("error"), failure.to_string());
+          writeln!(
-          println!();
+            writer,
+            "{}: {}",
+            colors::red_bold("error"),
+            failure.to_string()
+          )
+          .unwrap();
+          writeln!(writer).unwrap();
           failure_titles.push(failure_title);
         }
       }
@@ -136,22 +146,24 @@ pub(super) fn report_summary(
           "{} (uncaught error)",
           to_relative_path_or_remote_url(cwd, &origin)
         );
-        println!("{}", &failure_title);
+        writeln!(writer, "{}", &failure_title).unwrap();
-        println!(
+        writeln!(
+          writer,
           "{}: {}",
           colors::red_bold("error"),
           format_test_error(js_error)
-        );
+        )
-        println!("This error was not caught from a test and caused the test runner to fail on the referenced module.");
+        .unwrap();
-        println!("It most likely originated from a dangling promise, event/timeout handler or top-level code.");
+        writeln!(writer, "This error was not caught from a test and caused the test runner to fail on the referenced module.").unwrap();
-        println!();
+        writeln!(writer, "It most likely originated from a dangling promise, event/timeout handler or top-level code.").unwrap();
+        writeln!(writer).unwrap();
         failure_titles.push(failure_title);
       }
     }
     // note: the trailing whitespace is intentional to get a red background
-    println!("{}\n", colors::white_bold_on_red(" FAILURES "));
+    writeln!(writer, "{}\n", colors::white_bold_on_red(" FAILURES ")).unwrap();
     for failure_title in failure_titles {
-      println!("{failure_title}");
+      writeln!(writer, "{failure_title}").unwrap();
     }
   }
 
@@ -201,10 +213,12 @@ pub(super) fn report_summary(
     write!(summary_result, " | {} filtered out", summary.filtered_out).unwrap()
   };
 
-  println!(
+  writeln!(
+    writer,
     "\n{} | {} {}\n",
     status,
     summary_result,
     colors::gray(format!("({})", display::human_elapsed(elapsed.as_millis()))),
-  );
+  )
+  .unwrap();
 }

cli/tools/test/reporters/dot.rs

@@ -184,7 +184,12 @@ impl TestReporter for DotTestReporter {
     _tests: &IndexMap<usize, TestDescription>,
     _test_steps: &IndexMap<usize, TestStepDescription>,
   ) {
-    common::report_summary(&self.cwd, &self.summary, elapsed);
+    common::report_summary(
+      &mut std::io::stdout(),
+      &self.cwd,
+      &self.summary,
+      elapsed,
+    );
   }
 
   fn report_sigint(
@@ -193,7 +198,13 @@ impl TestReporter for DotTestReporter {
     tests: &IndexMap<usize, TestDescription>,
     test_steps: &IndexMap<usize, TestStepDescription>,
   ) {
-    common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
+    common::report_sigint(
+      &mut std::io::stdout(),
+      &self.cwd,
+      tests_pending,
+      tests,
+      test_steps,
+    );
   }
 
   fn flush_report(

cli/tools/test/reporters/pretty.rs

@@ -9,6 +9,7 @@ pub struct PrettyTestReporter {
   echo_output: bool,
   in_new_line: bool,
   filter: bool,
+  repl: bool,
   scope_test_id: Option<usize>,
   cwd: Url,
   did_have_user_output: bool,
@@ -16,6 +17,7 @@ pub struct PrettyTestReporter {
   child_results_buffer:
     HashMap<usize, IndexMap<usize, (TestStepDescription, TestStepResult, u64)>>,
   summary: TestSummary,
+  writer: Box<dyn std::io::Write>,
 }
 
 impl PrettyTestReporter {
@@ -23,35 +25,40 @@ impl PrettyTestReporter {
     parallel: bool,
     echo_output: bool,
     filter: bool,
+    repl: bool,
   ) -> PrettyTestReporter {
     PrettyTestReporter {
       parallel,
       echo_output,
       in_new_line: true,
       filter,
+      repl,
       scope_test_id: None,
       cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(),
       did_have_user_output: false,
       started_tests: false,
       child_results_buffer: Default::default(),
       summary: TestSummary::new(),
+      writer: Box::new(std::io::stdout()),
     }
   }
 
   fn force_report_wait(&mut self, description: &TestDescription) {
     if !self.in_new_line {
-      println!();
+      writeln!(&mut self.writer).unwrap();
     }
     if self.parallel {
-      print!(
+      write!(
+        &mut self.writer,
         "{}",
         colors::gray(format!(
           "{} => ",
           to_relative_path_or_remote_url(&self.cwd, &description.origin)
         ))
-      );
+      )
+      .unwrap();
     }
-    print!("{} ...", description.name);
+    write!(&mut self.writer, "{} ...", description.name).unwrap();
     self.in_new_line = false;
     // flush for faster feedback when line buffered
     std::io::stdout().flush().unwrap();
@@ -61,9 +68,15 @@ impl PrettyTestReporter {
   fn force_report_step_wait(&mut self, description: &TestStepDescription) {
     self.write_output_end();
     if !self.in_new_line {
-      println!();
+      writeln!(&mut self.writer).unwrap();
     }
-    print!("{}{} ...", "  ".repeat(description.level), description.name);
+    write!(
+      &mut self.writer,
+      "{}{} ...",
+      "  ".repeat(description.level),
+      description.name
+    )
+    .unwrap();
     self.in_new_line = false;
     // flush for faster feedback when line buffered
     std::io::stdout().flush().unwrap();
@@ -99,19 +112,21 @@ impl PrettyTestReporter {
       TestStepResult::Ignored => colors::yellow("ignored").to_string(),
       TestStepResult::Failed(failure) => failure.format_label(),
     };
-    print!(" {}", status);
+    write!(&mut self.writer, " {}", status).unwrap();
     if let TestStepResult::Failed(failure) = result {
       if let Some(inline_summary) = failure.format_inline_summary() {
-        print!(" ({})", inline_summary)
+        write!(&mut self.writer, " ({})", inline_summary).unwrap()
       }
     }
     if !matches!(result, TestStepResult::Failed(TestFailure::Incomplete)) {
-      print!(
+      write!(
+        &mut self.writer,
         " {}",
         colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
-      );
+      )
+      .unwrap();
     }
-    println!();
+    writeln!(&mut self.writer).unwrap();
     self.in_new_line = true;
     if self.parallel {
       self.scope_test_id = None;
@@ -127,7 +142,12 @@ impl PrettyTestReporter {
 
   fn write_output_end(&mut self) {
     if self.did_have_user_output {
-      println!("{}", colors::gray("----- output end -----"));
+      writeln!(
+        &mut self.writer,
+        "{}",
+        colors::gray("----- output end -----")
+      )
+      .unwrap();
       self.in_new_line = true;
       self.did_have_user_output = false;
     }
@@ -139,11 +159,15 @@ impl TestReporter for PrettyTestReporter {
   fn report_plan(&mut self, plan: &TestPlan) {
     self.summary.total += plan.total;
     self.summary.filtered_out += plan.filtered_out;
+    if self.repl {
+      return;
+    }
     if self.parallel || (self.filter && plan.total == 0) {
       return;
     }
     let inflection = if plan.total == 1 { "test" } else { "tests" };
-    println!(
+    writeln!(
+      &mut self.writer,
       "{}",
       colors::gray(format!(
         "running {} {} from {}",
@@ -151,7 +175,8 @@ impl TestReporter for PrettyTestReporter {
         inflection,
         to_relative_path_or_remote_url(&self.cwd, &plan.origin)
       ))
-    );
+    )
+    .unwrap();
     self.in_new_line = true;
   }
 
@@ -170,9 +195,14 @@ impl TestReporter for PrettyTestReporter {
     if !self.did_have_user_output && self.started_tests {
       self.did_have_user_output = true;
       if !self.in_new_line {
-        println!();
+        writeln!(&mut self.writer).unwrap();
       }
-      println!("{}", colors::gray("------- output -------"));
+      writeln!(
+        &mut self.writer,
+        "{}",
+        colors::gray("------- output -------")
+      )
+      .unwrap();
       self.in_new_line = true;
     }
 
@@ -221,16 +251,18 @@ impl TestReporter for PrettyTestReporter {
       TestResult::Failed(failure) => failure.format_label(),
       TestResult::Cancelled => colors::gray("cancelled").to_string(),
     };
-    print!(" {}", status);
+    write!(&mut self.writer, " {}", status).unwrap();
     if let TestResult::Failed(failure) = result {
       if let Some(inline_summary) = failure.format_inline_summary() {
-        print!(" ({})", inline_summary)
+        write!(&mut self.writer, " ({})", inline_summary).unwrap();
       }
     }
-    println!(
+    writeln!(
+      &mut self.writer,
       " {}",
       colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
-    );
+    )
+    .unwrap();
     self.in_new_line = true;
     self.scope_test_id = None;
   }
@@ -243,13 +275,15 @@ impl TestReporter for PrettyTestReporter {
       .push((origin.to_string(), error));
 
     if !self.in_new_line {
-      println!();
+      writeln!(&mut self.writer).unwrap();
     }
-    println!(
+    writeln!(
+      &mut self.writer,
       "Uncaught error from {} {}",
       to_relative_path_or_remote_url(&self.cwd, origin),
       colors::red("FAILED")
-    );
+    )
+    .unwrap();
     self.in_new_line = true;
     self.did_have_user_output = false;
   }
@@ -295,14 +329,16 @@ impl TestReporter for PrettyTestReporter {
 
     if self.parallel {
       self.write_output_end();
-      print!(
+      write!(
+        &mut self.writer,
         "{} {} ...",
         colors::gray(format!(
           "{} =>",
           to_relative_path_or_remote_url(&self.cwd, &desc.origin)
         )),
         common::format_test_step_ancestry(desc, tests, test_steps)
-      );
+      )
+      .unwrap();
       self.in_new_line = false;
       self.scope_test_id = Some(desc.id);
       self.force_report_step_result(desc, result, elapsed);
@@ -331,7 +367,7 @@ impl TestReporter for PrettyTestReporter {
     _tests: &IndexMap<usize, TestDescription>,
     _test_steps: &IndexMap<usize, TestStepDescription>,
   ) {
-    common::report_summary(&self.cwd, &self.summary, elapsed);
+    common::report_summary(&mut self.writer, &self.cwd, &self.summary, elapsed);
     self.in_new_line = true;
   }
 
@@ -341,7 +377,13 @@ impl TestReporter for PrettyTestReporter {
     tests: &IndexMap<usize, TestDescription>,
     test_steps: &IndexMap<usize, TestStepDescription>,
   ) {
-    common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
+    common::report_sigint(
+      &mut self.writer,
+      &self.cwd,
+      tests_pending,
+      tests,
+      test_steps,
+    );
     self.in_new_line = true;
   }
 
@@ -351,6 +393,7 @@ impl TestReporter for PrettyTestReporter {
     _tests: &IndexMap<usize, TestDescription>,
     _test_steps: &IndexMap<usize, TestStepDescription>,
   ) -> anyhow::Result<()> {
+    self.writer.flush().unwrap();
     Ok(())
   }
 }

cli/tools/test/reporters/tap.rs

@@ -227,7 +227,13 @@ impl TestReporter for TapTestReporter {
     test_steps: &IndexMap<usize, TestStepDescription>,
   ) {
     println!("Bail out! SIGINT received.");
-    common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
+    common::report_sigint(
+      &mut std::io::stdout(),
+      &self.cwd,
+      tests_pending,
+      tests,
+      test_steps,
+    );
   }
 
   fn flush_report(