1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-30 11:16:38 -05:00
denoland-deno/cli/tools/test/reporters/pretty.rs

424 lines
11 KiB
Rust
Raw Normal View History

// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use super::common;
use super::fmt::to_relative_path_or_remote_url;
use super::*;
pub struct PrettyTestReporter {
parallel: bool,
echo_output: bool,
in_new_line: bool,
phase: &'static str,
filter: bool,
repl: bool,
scope_test_id: Option<usize>,
cwd: Url,
did_have_user_output: bool,
started_tests: bool,
ended_tests: bool,
child_results_buffer:
HashMap<usize, IndexMap<usize, (TestStepDescription, TestStepResult, u64)>>,
summary: TestSummary,
writer: Box<dyn std::io::Write>,
}
impl PrettyTestReporter {
pub fn new(
parallel: bool,
echo_output: bool,
filter: bool,
repl: bool,
fix(cli): output more detailed information for steps when using JUnit reporter (#22797) This patch gets JUnit reporter to output more detailed information for test steps (subtests). ## Issue with previous implementation In the previous implementation, the test hierarchy was represented using several XML tags like the following: - `<testsuites>` corresponds to the entire test (one execution of `deno test` has exactly one `<testsuites>` tag) - `<testsuite>` corresponds to one file, such as `main_test.ts` - `<testcase>` corresponds to one `Deno.test(...)` - `<property>` corresponds to one `t.step(...)` This structure describes the test layers but one problem is that `<property>` tag is used for any use cases so some tools that can ingest a JUnit XML file might not be able to interpret `<property>` as subtests. ## How other tools address it Some of the testing frameworks in the ecosystem address this issue by fitting subtests into the `<testcase>` layer. For instance, take a look at the following Go test file: ```go package main_test import "testing" func TestMain(t *testing.T) { t.Run("child 1", func(t *testing.T) { // OK }) t.Run("child 2", func(t *testing.T) { // Error t.Fatal("error") }) } ``` Running [gotestsum], we can get the output like this: ```xml <?xml version="1.0" encoding="UTF-8"?> <testsuites tests="3" failures="2" errors="0" time="1.013694"> <testsuite tests="3" failures="2" time="0.510000" name="example/gosumtest" timestamp="2024-03-11T12:26:39+09:00"> <properties> <property name="go.version" value="go1.22.1 darwin/arm64"></property> </properties> <testcase classname="example/gosumtest" name="TestMain/child_2" time="0.000000"> <failure message="Failed" type="">=== RUN TestMain/child_2&#xA; main_test.go:12: error&#xA;--- FAIL: TestMain/child_2 (0.00s)&#xA;</failure> </testcase> <testcase classname="example/gosumtest" name="TestMain" time="0.000000"> <failure message="Failed" type="">=== RUN TestMain&#xA;--- FAIL: TestMain (0.00s)&#xA;</failure> </testcase> <testcase classname="example/gosumtest" name="TestMain/child_1" time="0.000000"></testcase> </testsuite> </testsuites> ``` This output shows that nested test cases are squashed into the `<testcase>` layer by treating them as the same layer as their parent, `TestMain`. We can still distinguish nested ones by their `name` attributes that look like `TestMain/<subtest_name>`. As described in #22795, [vitest] solves the issue in the same way as [gotestsum]. One downside of this would be that one test failure that happens in a nested test case will end up being counted multiple times, because not only the subtest but also its wrapping container(s) are considered to be failures. In fact, in the [gotestsum] output above, `TestMain/child_2` failed (which is totally expected) while its parent, `TestMain`, was also counted as failure. As https://github.com/denoland/deno/pull/20273#discussion_r1307558757 pointed out, there is a test runner that offers flexibility to prevent this, but I personally don't think the "duplicate failure count" issue is a big deal. ## How to fix the issue in this patch This patch fixes the issue with the same approach as [gotestsum] and [vitest]. More specifically, nested test cases are put into the `<testcase>` level and their names are now represented as squashed test names concatenated by `>` (e.g. `parent 2 > child 1 > grandchild 1`). This change also allows us to put a detailed error message as `<failure>` tag within the `<testcase>` tag, which should be handled nicely by third-party tools supporting JUnit XML. ## Extra fix Also, file paths embedded into XML outputs are changed from absolute path to relative path, which is helpful when running the test suites in several different environments like CI. Resolves #22795 [gotestsum]: https://github.com/gotestyourself/gotestsum [vitest]: https://vitest.dev/ --------- Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>
2024-03-25 11:08:46 -04:00
cwd: Url,
) -> PrettyTestReporter {
PrettyTestReporter {
parallel,
echo_output,
in_new_line: true,
phase: "",
filter,
repl,
scope_test_id: None,
fix(cli): output more detailed information for steps when using JUnit reporter (#22797) This patch gets JUnit reporter to output more detailed information for test steps (subtests). ## Issue with previous implementation In the previous implementation, the test hierarchy was represented using several XML tags like the following: - `<testsuites>` corresponds to the entire test (one execution of `deno test` has exactly one `<testsuites>` tag) - `<testsuite>` corresponds to one file, such as `main_test.ts` - `<testcase>` corresponds to one `Deno.test(...)` - `<property>` corresponds to one `t.step(...)` This structure describes the test layers but one problem is that `<property>` tag is used for any use cases so some tools that can ingest a JUnit XML file might not be able to interpret `<property>` as subtests. ## How other tools address it Some of the testing frameworks in the ecosystem address this issue by fitting subtests into the `<testcase>` layer. For instance, take a look at the following Go test file: ```go package main_test import "testing" func TestMain(t *testing.T) { t.Run("child 1", func(t *testing.T) { // OK }) t.Run("child 2", func(t *testing.T) { // Error t.Fatal("error") }) } ``` Running [gotestsum], we can get the output like this: ```xml <?xml version="1.0" encoding="UTF-8"?> <testsuites tests="3" failures="2" errors="0" time="1.013694"> <testsuite tests="3" failures="2" time="0.510000" name="example/gosumtest" timestamp="2024-03-11T12:26:39+09:00"> <properties> <property name="go.version" value="go1.22.1 darwin/arm64"></property> </properties> <testcase classname="example/gosumtest" name="TestMain/child_2" time="0.000000"> <failure message="Failed" type="">=== RUN TestMain/child_2&#xA; main_test.go:12: error&#xA;--- FAIL: TestMain/child_2 (0.00s)&#xA;</failure> </testcase> <testcase classname="example/gosumtest" name="TestMain" time="0.000000"> <failure message="Failed" type="">=== RUN TestMain&#xA;--- FAIL: TestMain (0.00s)&#xA;</failure> </testcase> <testcase classname="example/gosumtest" name="TestMain/child_1" time="0.000000"></testcase> </testsuite> </testsuites> ``` This output shows that nested test cases are squashed into the `<testcase>` layer by treating them as the same layer as their parent, `TestMain`. We can still distinguish nested ones by their `name` attributes that look like `TestMain/<subtest_name>`. As described in #22795, [vitest] solves the issue in the same way as [gotestsum]. One downside of this would be that one test failure that happens in a nested test case will end up being counted multiple times, because not only the subtest but also its wrapping container(s) are considered to be failures. In fact, in the [gotestsum] output above, `TestMain/child_2` failed (which is totally expected) while its parent, `TestMain`, was also counted as failure. As https://github.com/denoland/deno/pull/20273#discussion_r1307558757 pointed out, there is a test runner that offers flexibility to prevent this, but I personally don't think the "duplicate failure count" issue is a big deal. ## How to fix the issue in this patch This patch fixes the issue with the same approach as [gotestsum] and [vitest]. More specifically, nested test cases are put into the `<testcase>` level and their names are now represented as squashed test names concatenated by `>` (e.g. `parent 2 > child 1 > grandchild 1`). This change also allows us to put a detailed error message as `<failure>` tag within the `<testcase>` tag, which should be handled nicely by third-party tools supporting JUnit XML. ## Extra fix Also, file paths embedded into XML outputs are changed from absolute path to relative path, which is helpful when running the test suites in several different environments like CI. Resolves #22795 [gotestsum]: https://github.com/gotestyourself/gotestsum [vitest]: https://vitest.dev/ --------- Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>
2024-03-25 11:08:46 -04:00
cwd,
did_have_user_output: false,
started_tests: false,
ended_tests: false,
child_results_buffer: Default::default(),
summary: TestSummary::new(),
writer: Box::new(std::io::stdout()),
}
}
pub fn with_writer(self, writer: Box<dyn std::io::Write>) -> Self {
Self { writer, ..self }
}
fn force_report_wait(&mut self, description: &TestDescription) {
if !self.in_new_line {
writeln!(&mut self.writer).unwrap();
}
if self.parallel {
write!(
&mut self.writer,
"{}",
colors::gray(format!(
"{} => ",
to_relative_path_or_remote_url(&self.cwd, &description.origin)
))
)
.unwrap();
}
write!(&mut self.writer, "{} ...", description.name).unwrap();
self.in_new_line = false;
// flush for faster feedback when line buffered
std::io::stdout().flush().unwrap();
self.scope_test_id = Some(description.id);
}
fn force_report_step_wait(&mut self, description: &TestStepDescription) {
self.write_output_end();
if !self.in_new_line {
writeln!(&mut self.writer).unwrap();
}
write!(
&mut self.writer,
"{}{} ...",
" ".repeat(description.level),
description.name
)
.unwrap();
self.in_new_line = false;
// flush for faster feedback when line buffered
std::io::stdout().flush().unwrap();
self.scope_test_id = Some(description.id);
}
fn force_report_step_result(
&mut self,
description: &TestStepDescription,
result: &TestStepResult,
elapsed: u64,
) {
self.write_output_end();
if self.in_new_line || self.scope_test_id != Some(description.id) {
self.force_report_step_wait(description);
}
if !self.parallel {
let child_results = self
.child_results_buffer
.remove(&description.id)
.unwrap_or_default();
for (desc, result, elapsed) in child_results.values() {
self.force_report_step_result(desc, result, *elapsed);
}
if !child_results.is_empty() {
self.force_report_step_wait(description);
}
}
let status = match &result {
TestStepResult::Ok => colors::green("ok").to_string(),
TestStepResult::Ignored => colors::yellow("ignored").to_string(),
TestStepResult::Failed(failure) => failure.format_label(),
};
write!(&mut self.writer, " {}", status).unwrap();
if let TestStepResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
write!(&mut self.writer, " ({})", inline_summary).unwrap()
}
}
if !matches!(result, TestStepResult::Failed(TestFailure::Incomplete)) {
write!(
&mut self.writer,
" {}",
colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
)
.unwrap();
}
writeln!(&mut self.writer).unwrap();
self.in_new_line = true;
if self.parallel {
self.scope_test_id = None;
} else {
self.scope_test_id = Some(description.parent_id);
}
self
.child_results_buffer
.entry(description.parent_id)
.or_default()
.shift_remove(&description.id);
}
fn write_output_end(&mut self) {
if self.did_have_user_output {
writeln!(
&mut self.writer,
"{}",
colors::gray(format!("----- {}output end -----", self.phase))
)
.unwrap();
self.in_new_line = true;
self.did_have_user_output = false;
}
}
}
impl TestReporter for PrettyTestReporter {
fn report_register(&mut self, _description: &TestDescription) {}
fn report_plan(&mut self, plan: &TestPlan) {
self.write_output_end();
self.summary.total += plan.total;
self.summary.filtered_out += plan.filtered_out;
if self.repl {
return;
}
if self.parallel || (self.filter && plan.total == 0) {
return;
}
let inflection = if plan.total == 1 { "test" } else { "tests" };
writeln!(
&mut self.writer,
"{}",
colors::gray(format!(
"running {} {} from {}",
plan.total,
inflection,
to_relative_path_or_remote_url(&self.cwd, &plan.origin)
))
)
.unwrap();
self.in_new_line = true;
}
fn report_wait(&mut self, description: &TestDescription) {
if !self.parallel {
self.force_report_wait(description);
}
self.started_tests = true;
}
fn report_output(&mut self, output: &[u8]) {
if !self.echo_output {
return;
}
if !self.did_have_user_output {
self.did_have_user_output = true;
if !self.in_new_line {
writeln!(&mut self.writer).unwrap();
}
self.phase = if !self.started_tests {
"pre-test "
} else if self.ended_tests {
"post-test "
} else {
""
};
writeln!(
&mut self.writer,
"{}",
colors::gray(format!("------- {}output -------", self.phase))
)
.unwrap();
self.in_new_line = true;
}
// output everything to stdout in order to prevent
// stdout and stderr racing
std::io::stdout().write_all(output).unwrap();
}
fn report_result(
&mut self,
description: &TestDescription,
result: &TestResult,
elapsed: u64,
) {
match &result {
TestResult::Ok => {
self.summary.passed += 1;
}
TestResult::Ignored => {
self.summary.ignored += 1;
}
TestResult::Failed(failure) => {
self.summary.failed += 1;
self
.summary
.failures
.push((description.into(), failure.clone()));
}
TestResult::Cancelled => {
self.summary.failed += 1;
}
}
if self.parallel {
self.force_report_wait(description);
}
self.write_output_end();
if self.in_new_line || self.scope_test_id != Some(description.id) {
self.force_report_wait(description);
}
let status = match result {
TestResult::Ok => colors::green("ok").to_string(),
TestResult::Ignored => colors::yellow("ignored").to_string(),
TestResult::Failed(failure) => failure.format_label(),
TestResult::Cancelled => colors::gray("cancelled").to_string(),
};
write!(&mut self.writer, " {}", status).unwrap();
if let TestResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
write!(&mut self.writer, " ({})", inline_summary).unwrap();
}
}
writeln!(
&mut self.writer,
" {}",
colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
)
.unwrap();
self.in_new_line = true;
self.scope_test_id = None;
}
fn report_uncaught_error(&mut self, origin: &str, error: Box<JsError>) {
self.summary.failed += 1;
self
.summary
.uncaught_errors
.push((origin.to_string(), error));
if !self.in_new_line {
writeln!(&mut self.writer).unwrap();
}
writeln!(
&mut self.writer,
"Uncaught error from {} {}",
to_relative_path_or_remote_url(&self.cwd, origin),
colors::red("FAILED")
)
.unwrap();
self.in_new_line = true;
self.did_have_user_output = false;
}
fn report_step_register(&mut self, _description: &TestStepDescription) {}
fn report_step_wait(&mut self, description: &TestStepDescription) {
if !self.parallel && self.scope_test_id == Some(description.parent_id) {
self.force_report_step_wait(description);
}
}
fn report_step_result(
&mut self,
desc: &TestStepDescription,
result: &TestStepResult,
elapsed: u64,
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) {
match &result {
TestStepResult::Ok => {
self.summary.passed_steps += 1;
}
TestStepResult::Ignored => {
self.summary.ignored_steps += 1;
}
TestStepResult::Failed(failure) => {
self.summary.failed_steps += 1;
self.summary.failures.push((
TestFailureDescription {
id: desc.id,
name: common::format_test_step_ancestry(desc, tests, test_steps),
origin: desc.origin.clone(),
location: desc.location.clone(),
},
failure.clone(),
))
}
}
if self.parallel {
self.write_output_end();
write!(
&mut self.writer,
"{} {} ...",
colors::gray(format!(
"{} =>",
to_relative_path_or_remote_url(&self.cwd, &desc.origin)
)),
common::format_test_step_ancestry(desc, tests, test_steps)
)
.unwrap();
self.in_new_line = false;
self.scope_test_id = Some(desc.id);
self.force_report_step_result(desc, result, elapsed);
} else {
let sibling_results =
self.child_results_buffer.entry(desc.parent_id).or_default();
if self.scope_test_id == Some(desc.id)
|| self.scope_test_id == Some(desc.parent_id)
{
let sibling_results = std::mem::take(sibling_results);
self.force_report_step_result(desc, result, elapsed);
// Flush buffered sibling results.
for (desc, result, elapsed) in sibling_results.values() {
self.force_report_step_result(desc, result, *elapsed);
}
} else {
sibling_results
.insert(desc.id, (desc.clone(), result.clone(), elapsed));
}
}
}
fn report_summary(
&mut self,
elapsed: &Duration,
_tests: &IndexMap<usize, TestDescription>,
_test_steps: &IndexMap<usize, TestStepDescription>,
) {
self.write_output_end();
common::report_summary(&mut self.writer, &self.cwd, &self.summary, elapsed);
if !self.repl {
writeln!(&mut self.writer).unwrap();
}
self.in_new_line = true;
}
fn report_sigint(
&mut self,
tests_pending: &HashSet<usize>,
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) {
common::report_sigint(
&mut self.writer,
&self.cwd,
tests_pending,
tests,
test_steps,
);
self.in_new_line = true;
}
fn report_completed(&mut self) {
self.write_output_end();
self.ended_tests = true;
}
fn flush_report(
&mut self,
_elapsed: &Duration,
_tests: &IndexMap<usize, TestDescription>,
_test_steps: &IndexMap<usize, TestStepDescription>,
) -> anyhow::Result<()> {
self.writer.flush().unwrap();
Ok(())
}
}