foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-24 15:19:26 -05:00
Code Issues Packages 1 Wiki Activity

refactor(test): support custom writer in PrettyTestReporter (#20783)

Browse source
This commit is contained in:
Nayeem Rahman 2023-10-05 11:25:15 +01:00 committed by GitHub
parent fd4fc2d818
commit 551a081450
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 291 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cli/lsp/testing/execution.rs
View file

@ -404,6 +404,7 @@ impl TestRun {
);
}
}
test::TestEvent::ForceEndReport => {}
test::TestEvent::Sigint => {}
}
}

4
cli/tools/test/fmt.rs
View file

@ -3,7 +3,9 @@
use super::*;
pub fn to_relative_path_or_remote_url(cwd: &Url, path_or_url: &str) -> String {
let url = Url::parse(path_or_url).unwrap();
let Ok(url) = Url::parse(path_or_url) else {
return "<anonymous>".to_string();
};
if url.scheme() == "file" {
if let Some(mut r) = cwd.make_relative(&url) {
if !r.starts_with("../") {

307
cli/tools/test/mod.rs
View file

@ -50,6 +50,7 @@ use deno_runtime::fmt_errors::format_js_error;
use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsContainer;
use deno_runtime::tokio_util::create_and_run_current_thread;
use deno_runtime::worker::MainWorker;
use indexmap::IndexMap;
use indexmap::IndexSet;
use log::Level;
@ -77,11 +78,12 @@ use std::time::Instant;
use std::time::SystemTime;
use tokio::signal;
use tokio::sync::mpsc::unbounded_channel;
use tokio::sync::mpsc::UnboundedReceiver;
use tokio::sync::mpsc::UnboundedSender;
use tokio::sync::mpsc::WeakUnboundedSender;
pub mod fmt;
mod reporters;
pub mod reporters;
pub use fmt::format_test_error;
use reporters::CompoundTestReporter;
@ -313,6 +315,7 @@ pub enum TestEvent {
StepRegister(TestStepDescription),
StepWait(usize),
StepResult(usize, TestStepResult, u64),
ForceEndReport,
Sigint,
}
@ -342,7 +345,7 @@ struct TestSpecifiersOptions {
junit_path: Option<String>,
}
#[derive(Debug, Clone)]
#[derive(Debug, Default, Clone)]
pub struct TestSpecifierOptions {
pub shuffle: Option<u64>,
pub filter: TestFilter,
@ -379,6 +382,7 @@ fn get_test_reporter(options: &TestSpecifiersOptions) -> Box<dyn TestReporter> {
parallel,
options.log_level != Some(Level::Error),
options.filter,
false,
)),
TestReporterConfig::Junit => {
Box::new(JunitTestReporter::new("-".to_string()))
@ -453,10 +457,35 @@ pub async fn test_specifier(
worker.dispatch_load_event(located_script_name!())?;
let tests = {
run_tests_for_worker(&mut worker, &specifier, &options, &fail_fast_tracker)
.await?;
// Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
// event loop to continue beyond what's needed to await results.
worker.dispatch_beforeunload_event(located_script_name!())?;
worker.dispatch_unload_event(located_script_name!())?;
if let Some(coverage_collector) = coverage_collector.as_mut() {
worker
.with_event_loop(coverage_collector.stop_collecting().boxed_local())
.await?;
}
Ok(())
}
pub async fn run_tests_for_worker(
worker: &mut MainWorker,
specifier: &ModuleSpecifier,
options: &TestSpecifierOptions,
fail_fast_tracker: &FailFastTracker,
) -> Result<(), AnyError> {
let (tests, mut sender) = {
let state_rc = worker.js_runtime.op_state();
let mut state = state_rc.borrow_mut();
std::mem::take(&mut state.borrow_mut::<ops::testing::TestContainer>().0)
(
std::mem::take(&mut state.borrow_mut::<ops::testing::TestContainer>().0),
state.borrow::<TestEventSender>().clone(),
)
};
let unfiltered = tests.len();
let tests = tests
@ -532,17 +561,6 @@ pub async fn test_specifier(
let elapsed = SystemTime::now().duration_since(earlier)?.as_millis();
sender.send(TestEvent::Result(desc.id, result, elapsed as u64))?;
}
// Ignore `defaultPrevented` of the `beforeunload` event. We don't allow the
// event loop to continue beyond what's needed to await results.
worker.dispatch_beforeunload_event(located_script_name!())?;
worker.dispatch_unload_event(located_script_name!())?;
if let Some(coverage_collector) = coverage_collector.as_mut() {
worker
.with_event_loop(coverage_collector.stop_collecting().boxed_local())
.await?;
}
Ok(())
}
@ -810,7 +828,7 @@ async fn test_specifiers(
specifiers
};
let (sender, mut receiver) = unbounded_channel::<TestEvent>();
let (sender, receiver) = unbounded_channel::<TestEvent>();
let sender = TestEventSender::new(sender);
let concurrent_jobs = options.concurrent_jobs;
@ -820,7 +838,7 @@ async fn test_specifiers(
sender_.upgrade().map(|s| s.send(TestEvent::Sigint).ok());
});
HAS_TEST_RUN_SIGINT_HANDLER.store(true, Ordering::Relaxed);
let mut reporter = get_test_reporter(&options);
let reporter = get_test_reporter(&options);
let fail_fast_tracker = FailFastTracker::new(options.fail_fast);
let join_handles = specifiers.into_iter().map(move |specifier| {
@ -840,144 +858,149 @@ async fn test_specifiers(
))
})
});
let join_stream = stream::iter(join_handles)
.buffer_unordered(concurrent_jobs.get())
.collect::<Vec<Result<Result<(), AnyError>, tokio::task::JoinError>>>();
let handler = {
spawn(async move {
let earlier = Instant::now();
let mut tests = IndexMap::new();
let mut test_steps = IndexMap::new();
let mut tests_started = HashSet::new();
let mut tests_with_result = HashSet::new();
let mut used_only = false;
let mut failed = false;
while let Some(event) = receiver.recv().await {
match event {
TestEvent::Register(description) => {
reporter.report_register(&description);
tests.insert(description.id, description);
}
TestEvent::Plan(plan) => {
if plan.used_only {
used_only = true;
}
reporter.report_plan(&plan);
}
TestEvent::Wait(id) => {
if tests_started.insert(id) {
reporter.report_wait(tests.get(&id).unwrap());
}
}
TestEvent::Output(output) => {
reporter.report_output(&output);
}
TestEvent::Result(id, result, elapsed) => {
if tests_with_result.insert(id) {
match result {
TestResult::Failed(_) | TestResult::Cancelled => {
failed = true;
}
_ => (),
}
reporter.report_result(tests.get(&id).unwrap(), &result, elapsed);
}
}
TestEvent::UncaughtError(origin, error) => {
failed = true;
reporter.report_uncaught_error(&origin, error);
}
TestEvent::StepRegister(description) => {
reporter.report_step_register(&description);
test_steps.insert(description.id, description);
}
TestEvent::StepWait(id) => {
if tests_started.insert(id) {
reporter.report_step_wait(test_steps.get(&id).unwrap());
}
}
TestEvent::StepResult(id, result, duration) => {
if tests_with_result.insert(id) {
reporter.report_step_result(
test_steps.get(&id).unwrap(),
&result,
duration,
&tests,
&test_steps,
);
}
}
TestEvent::Sigint => {
let elapsed = Instant::now().duration_since(earlier);
reporter.report_sigint(
&tests_started
.difference(&tests_with_result)
.copied()
.collect(),
&tests,
&test_steps,
);
if let Err(err) =
reporter.flush_report(&elapsed, &tests, &test_steps)
{
eprint!("Test reporter failed to flush: {}", err)
}
std::process::exit(130);
}
}
}
sigint_handler_handle.abort();
HAS_TEST_RUN_SIGINT_HANDLER.store(false, Ordering::Relaxed);
let elapsed = Instant::now().duration_since(earlier);
reporter.report_summary(&elapsed, &tests, &test_steps);
if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) {
return Err(generic_error(format!(
"Test reporter failed to flush: {}",
err
)));
}
if used_only {
return Err(generic_error(
"Test failed because the \"only\" option was used",
));
}
if failed {
return Err(generic_error("Test failed"));
}
Ok(())
})
};
let handler = spawn(async move { report_tests(receiver, reporter).await.0 });
let (join_results, result) = future::join(join_stream, handler).await;
// propagate any errors
sigint_handler_handle.abort();
HAS_TEST_RUN_SIGINT_HANDLER.store(false, Ordering::Relaxed);
for join_result in join_results {
join_result??;
}
result??;
Ok(())
}
/// Gives receiver back in case it was ended with `TestEvent::ForceEndReport`.
pub async fn report_tests(
mut receiver: UnboundedReceiver<TestEvent>,
mut reporter: Box<dyn TestReporter>,
) -> (Result<(), AnyError>, UnboundedReceiver<TestEvent>) {
let mut tests = IndexMap::new();
let mut test_steps = IndexMap::new();
let mut tests_started = HashSet::new();
let mut tests_with_result = HashSet::new();
let mut start_time = None;
let mut had_plan = false;
let mut used_only = false;
let mut failed = false;
while let Some(event) = receiver.recv().await {
match event {
TestEvent::Register(description) => {
reporter.report_register(&description);
tests.insert(description.id, description);
}
TestEvent::Plan(plan) => {
if !had_plan {
start_time = Some(Instant::now());
had_plan = true;
}
if plan.used_only {
used_only = true;
}
reporter.report_plan(&plan);
}
TestEvent::Wait(id) => {
if tests_started.insert(id) {
reporter.report_wait(tests.get(&id).unwrap());
}
}
TestEvent::Output(output) => {
reporter.report_output(&output);
}
TestEvent::Result(id, result, elapsed) => {
if tests_with_result.insert(id) {
match result {
TestResult::Failed(_) | TestResult::Cancelled => {
failed = true;
}
_ => (),
}
reporter.report_result(tests.get(&id).unwrap(), &result, elapsed);
}
}
TestEvent::UncaughtError(origin, error) => {
failed = true;
reporter.report_uncaught_error(&origin, error);
}
TestEvent::StepRegister(description) => {
reporter.report_step_register(&description);
test_steps.insert(description.id, description);
}
TestEvent::StepWait(id) => {
if tests_started.insert(id) {
reporter.report_step_wait(test_steps.get(&id).unwrap());
}
}
TestEvent::StepResult(id, result, duration) => {
if tests_with_result.insert(id) {
reporter.report_step_result(
test_steps.get(&id).unwrap(),
&result,
duration,
&tests,
&test_steps,
);
}
}
TestEvent::ForceEndReport => {
break;
}
TestEvent::Sigint => {
let elapsed = start_time
.map(|t| Instant::now().duration_since(t))
.unwrap_or_default();
reporter.report_sigint(
&tests_started
.difference(&tests_with_result)
.copied()
.collect(),
&tests,
&test_steps,
);
if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) {
eprint!("Test reporter failed to flush: {}", err)
}
std::process::exit(130);
}
}
}
let elapsed = start_time
.map(|t| Instant::now().duration_since(t))
.unwrap_or_default();
reporter.report_summary(&elapsed, &tests, &test_steps);
if let Err(err) = reporter.flush_report(&elapsed, &tests, &test_steps) {
return (
Err(generic_error(format!(
"Test reporter failed to flush: {}",
err
))),
receiver,
);
}
if used_only {
return (
Err(generic_error(
"Test failed because the \"only\" option was used",
)),
receiver,
);
}
if failed {
return (Err(generic_error("Test failed")), receiver);
}
(Ok(()), receiver)
}
/// Checks if the path has a basename and extension Deno supports for tests.
pub(crate) fn is_supported_test_path(path: &Path) -> bool {
if let Some(name) = path.file_stem() {
@ -1300,7 +1323,7 @@ pub async fn run_tests_with_watch(
/// Tracks failures for the `--fail-fast` argument in
/// order to tell when to stop running tests.
#[derive(Clone)]
#[derive(Clone, Default)]
pub struct FailFastTracker {
max_count: Option<usize>,
failure_count: Arc<AtomicUsize>,

50
cli/tools/test/reporters/common.rs
View file

@ -66,6 +66,7 @@ pub fn format_test_step_for_summary(
}
pub(super) fn report_sigint(
writer: &mut dyn std::io::Write,
cwd: &Url,
tests_pending: &HashSet<usize>,
tests: &IndexMap<usize, TestDescription>,
@ -84,17 +85,20 @@ pub(super) fn report_sigint(
.insert(format_test_step_for_summary(cwd, desc, tests, test_steps));
}
}
println!(
writeln!(
writer,
"\n{} The following tests were pending:\n",
colors::intense_blue("SIGINT")
);
)
.unwrap();
for entry in formatted_pending {
println!("{}", entry);
writeln!(writer, "{}", entry).unwrap();
}
println!();
writeln!(writer).unwrap();
}
pub(super) fn report_summary(
writer: &mut dyn std::io::Write,
cwd: &Url,
summary: &TestSummary,
elapsed: &Duration,
@ -120,14 +124,20 @@ pub(super) fn report_summary(
}
// note: the trailing whitespace is intentional to get a red background
println!("\n{}\n", colors::white_bold_on_red(" ERRORS "));
writeln!(writer, "\n{}\n", colors::white_bold_on_red(" ERRORS ")).unwrap();
for (origin, (failures, uncaught_error)) in failures_by_origin {
for (description, failure) in failures {
if !failure.hide_in_summary() {
let failure_title = format_test_for_summary(cwd, description);
println!("{}", &failure_title);
println!("{}: {}", colors::red_bold("error"), failure.to_string());
println!();
writeln!(writer, "{}", &failure_title).unwrap();
writeln!(
writer,
"{}: {}",
colors::red_bold("error"),
failure.to_string()
)
.unwrap();
writeln!(writer).unwrap();
failure_titles.push(failure_title);
}
}
@ -136,22 +146,24 @@ pub(super) fn report_summary(
"{} (uncaught error)",
to_relative_path_or_remote_url(cwd, &origin)
);
println!("{}", &failure_title);
println!(
writeln!(writer, "{}", &failure_title).unwrap();
writeln!(
writer,
"{}: {}",
colors::red_bold("error"),
format_test_error(js_error)
);
println!("This error was not caught from a test and caused the test runner to fail on the referenced module.");
println!("It most likely originated from a dangling promise, event/timeout handler or top-level code.");
println!();
)
.unwrap();
writeln!(writer, "This error was not caught from a test and caused the test runner to fail on the referenced module.").unwrap();
writeln!(writer, "It most likely originated from a dangling promise, event/timeout handler or top-level code.").unwrap();
writeln!(writer).unwrap();
failure_titles.push(failure_title);
}
}
// note: the trailing whitespace is intentional to get a red background
println!("{}\n", colors::white_bold_on_red(" FAILURES "));
writeln!(writer, "{}\n", colors::white_bold_on_red(" FAILURES ")).unwrap();
for failure_title in failure_titles {
println!("{failure_title}");
writeln!(writer, "{failure_title}").unwrap();
}
}
@ -201,10 +213,12 @@ pub(super) fn report_summary(
write!(summary_result, " | {} filtered out", summary.filtered_out).unwrap()
};
println!(
writeln!(
writer,
"\n{} | {} {}\n",
status,
summary_result,
colors::gray(format!("({})", display::human_elapsed(elapsed.as_millis()))),
);
)
.unwrap();
}

15
cli/tools/test/reporters/dot.rs
View file

@ -184,7 +184,12 @@ impl TestReporter for DotTestReporter {
_tests: &IndexMap<usize, TestDescription>,
_test_steps: &IndexMap<usize, TestStepDescription>,
) {
common::report_summary(&self.cwd, &self.summary, elapsed);
common::report_summary(
&mut std::io::stdout(),
&self.cwd,
&self.summary,
elapsed,
);
}
fn report_sigint(
@ -193,7 +198,13 @@ impl TestReporter for DotTestReporter {
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) {
common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
common::report_sigint(
&mut std::io::stdout(),
&self.cwd,
tests_pending,
tests,
test_steps,
);
}
fn flush_report(

97
cli/tools/test/reporters/pretty.rs
View file

@ -9,6 +9,7 @@ pub struct PrettyTestReporter {
echo_output: bool,
in_new_line: bool,
filter: bool,
repl: bool,
scope_test_id: Option<usize>,
cwd: Url,
did_have_user_output: bool,
@ -16,6 +17,7 @@ pub struct PrettyTestReporter {
child_results_buffer:
HashMap<usize, IndexMap<usize, (TestStepDescription, TestStepResult, u64)>>,
summary: TestSummary,
writer: Box<dyn std::io::Write>,
}
impl PrettyTestReporter {
@ -23,35 +25,40 @@ impl PrettyTestReporter {
parallel: bool,
echo_output: bool,
filter: bool,
repl: bool,
) -> PrettyTestReporter {
PrettyTestReporter {
parallel,
echo_output,
in_new_line: true,
filter,
repl,
scope_test_id: None,
cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(),
did_have_user_output: false,
started_tests: false,
child_results_buffer: Default::default(),
summary: TestSummary::new(),
writer: Box::new(std::io::stdout()),
}
}
fn force_report_wait(&mut self, description: &TestDescription) {
if !self.in_new_line {
println!();
writeln!(&mut self.writer).unwrap();
}
if self.parallel {
print!(
write!(
&mut self.writer,
"{}",
colors::gray(format!(
"{} => ",
to_relative_path_or_remote_url(&self.cwd, &description.origin)
))
);
)
.unwrap();
}
print!("{} ...", description.name);
write!(&mut self.writer, "{} ...", description.name).unwrap();
self.in_new_line = false;
// flush for faster feedback when line buffered
std::io::stdout().flush().unwrap();
@ -61,9 +68,15 @@ impl PrettyTestReporter {
fn force_report_step_wait(&mut self, description: &TestStepDescription) {
self.write_output_end();
if !self.in_new_line {
println!();
writeln!(&mut self.writer).unwrap();
}
print!("{}{} ...", " ".repeat(description.level), description.name);
write!(
&mut self.writer,
"{}{} ...",
" ".repeat(description.level),
description.name
)
.unwrap();
self.in_new_line = false;
// flush for faster feedback when line buffered
std::io::stdout().flush().unwrap();
@ -99,19 +112,21 @@ impl PrettyTestReporter {
TestStepResult::Ignored => colors::yellow("ignored").to_string(),
TestStepResult::Failed(failure) => failure.format_label(),
};
print!(" {}", status);
write!(&mut self.writer, " {}", status).unwrap();
if let TestStepResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
print!(" ({})", inline_summary)
write!(&mut self.writer, " ({})", inline_summary).unwrap()
}
}
if !matches!(result, TestStepResult::Failed(TestFailure::Incomplete)) {
print!(
write!(
&mut self.writer,
" {}",
colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
);
)
.unwrap();
}
println!();
writeln!(&mut self.writer).unwrap();
self.in_new_line = true;
if self.parallel {
self.scope_test_id = None;
@ -127,7 +142,12 @@ impl PrettyTestReporter {
fn write_output_end(&mut self) {
if self.did_have_user_output {
println!("{}", colors::gray("----- output end -----"));
writeln!(
&mut self.writer,
"{}",
colors::gray("----- output end -----")
)
.unwrap();
self.in_new_line = true;
self.did_have_user_output = false;
}
@ -139,11 +159,15 @@ impl TestReporter for PrettyTestReporter {
fn report_plan(&mut self, plan: &TestPlan) {
self.summary.total += plan.total;
self.summary.filtered_out += plan.filtered_out;
if self.repl {
return;
}
if self.parallel || (self.filter && plan.total == 0) {
return;
}
let inflection = if plan.total == 1 { "test" } else { "tests" };
println!(
writeln!(
&mut self.writer,
"{}",
colors::gray(format!(
"running {} {} from {}",
@ -151,7 +175,8 @@ impl TestReporter for PrettyTestReporter {
inflection,
to_relative_path_or_remote_url(&self.cwd, &plan.origin)
))
);
)
.unwrap();
self.in_new_line = true;
}
@ -170,9 +195,14 @@ impl TestReporter for PrettyTestReporter {
if !self.did_have_user_output && self.started_tests {
self.did_have_user_output = true;
if !self.in_new_line {
println!();
writeln!(&mut self.writer).unwrap();
}
println!("{}", colors::gray("------- output -------"));
writeln!(
&mut self.writer,
"{}",
colors::gray("------- output -------")
)
.unwrap();
self.in_new_line = true;
}
@ -221,16 +251,18 @@ impl TestReporter for PrettyTestReporter {
TestResult::Failed(failure) => failure.format_label(),
TestResult::Cancelled => colors::gray("cancelled").to_string(),
};
print!(" {}", status);
write!(&mut self.writer, " {}", status).unwrap();
if let TestResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
print!(" ({})", inline_summary)
write!(&mut self.writer, " ({})", inline_summary).unwrap();
}
}
println!(
writeln!(
&mut self.writer,
" {}",
colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
);
)
.unwrap();
self.in_new_line = true;
self.scope_test_id = None;
}
@ -243,13 +275,15 @@ impl TestReporter for PrettyTestReporter {
.push((origin.to_string(), error));
if !self.in_new_line {
println!();
writeln!(&mut self.writer).unwrap();
}
println!(
writeln!(
&mut self.writer,
"Uncaught error from {} {}",
to_relative_path_or_remote_url(&self.cwd, origin),
colors::red("FAILED")
);
)
.unwrap();
self.in_new_line = true;
self.did_have_user_output = false;
}
@ -295,14 +329,16 @@ impl TestReporter for PrettyTestReporter {
if self.parallel {
self.write_output_end();
print!(
write!(
&mut self.writer,
"{} {} ...",
colors::gray(format!(
"{} =>",
to_relative_path_or_remote_url(&self.cwd, &desc.origin)
)),
common::format_test_step_ancestry(desc, tests, test_steps)
);
)
.unwrap();
self.in_new_line = false;
self.scope_test_id = Some(desc.id);
self.force_report_step_result(desc, result, elapsed);
@ -331,7 +367,7 @@ impl TestReporter for PrettyTestReporter {
_tests: &IndexMap<usize, TestDescription>,
_test_steps: &IndexMap<usize, TestStepDescription>,
) {
common::report_summary(&self.cwd, &self.summary, elapsed);
common::report_summary(&mut self.writer, &self.cwd, &self.summary, elapsed);
self.in_new_line = true;
}
@ -341,7 +377,13 @@ impl TestReporter for PrettyTestReporter {
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) {
common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
common::report_sigint(
&mut self.writer,
&self.cwd,
tests_pending,
tests,
test_steps,
);
self.in_new_line = true;
}
@ -351,6 +393,7 @@ impl TestReporter for PrettyTestReporter {
_tests: &IndexMap<usize, TestDescription>,
_test_steps: &IndexMap<usize, TestStepDescription>,
) -> anyhow::Result<()> {
self.writer.flush().unwrap();
Ok(())
}
}

8
cli/tools/test/reporters/tap.rs
View file

@ -227,7 +227,13 @@ impl TestReporter for TapTestReporter {
test_steps: &IndexMap<usize, TestStepDescription>,
) {
println!("Bail out! SIGINT received.");
common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
common::report_sigint(
&mut std::io::stdout(),
&self.cwd,
tests_pending,
tests,
test_steps,
);
}
fn flush_report(