foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-03 04:48:52 -05:00
Code Issues Packages 1 Wiki Activity

feat(test): print pending tests on sigint (#18246)

Browse source
This commit is contained in:
Nayeem Rahman 2023-03-25 19:32:11 +00:00 committed by GitHub
parent fe88b53e50
commit 8a4865c379
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 649 additions and 584 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

341
cli/js/40_testing.js
View file

@ -142,7 +142,8 @@ function assertOps(fn) {
const pre = core.metrics(); const pre = core.metrics();
const preTraces = new Map(core.opCallTraces); const preTraces = new Map(core.opCallTraces);
try { try {
await fn(desc); const innerResult = await fn(desc);
if (innerResult) return innerResult;
} finally { } finally {
// Defer until next event loop turn - that way timeouts and intervals // Defer until next event loop turn - that way timeouts and intervals
// cleared can actually be removed from resource table, otherwise // cleared can actually be removed from resource table, otherwise
@ -150,9 +151,6 @@ function assertOps(fn) {
await opSanitizerDelay(); await opSanitizerDelay();
await opSanitizerDelay(); await opSanitizerDelay();
} }
if (shouldSkipSanitizers(desc)) return;
const post = core.metrics(); const post = core.metrics();
const postTraces = new Map(core.opCallTraces); const postTraces = new Map(core.opCallTraces);
@ -161,7 +159,7 @@ function assertOps(fn) {
const dispatchedDiff = post.opsDispatchedAsync - pre.opsDispatchedAsync; const dispatchedDiff = post.opsDispatchedAsync - pre.opsDispatchedAsync;
const completedDiff = post.opsCompletedAsync - pre.opsCompletedAsync; const completedDiff = post.opsCompletedAsync - pre.opsCompletedAsync;
if (dispatchedDiff === completedDiff) return; if (dispatchedDiff === completedDiff) return null;
const details = []; const details = [];
for (const key in post.ops) { for (const key in post.ops) {
@ -215,19 +213,7 @@ function assertOps(fn) {
); );
} }
} }
return { failed: { leakedOps: [details, core.isOpCallTracingEnabled()] } };
let msg = `Test case is leaking async ops.
- ${ArrayPrototypeJoin(details, "\n - ")}`;
if (!core.isOpCallTracingEnabled()) {
msg +=
`\n\nTo get more details where ops were leaked, run again with --trace-ops flag.`;
} else {
msg += "\n";
}
throw assert(false, msg);
}; };
} }
@ -372,12 +358,8 @@ function assertResources(fn) {
/** @param desc {TestDescription | TestStepDescription} */ /** @param desc {TestDescription | TestStepDescription} */
return async function resourceSanitizer(desc) { return async function resourceSanitizer(desc) {
const pre = core.resources(); const pre = core.resources();
await fn(desc); const innerResult = await fn(desc);
if (innerResult) return innerResult;
if (shouldSkipSanitizers(desc)) {
return;
}
const post = core.resources(); const post = core.resources();
const allResources = new Set([ const allResources = new Set([
@ -404,14 +386,10 @@ function assertResources(fn) {
ArrayPrototypePush(details, detail); ArrayPrototypePush(details, detail);
} }
} }
if (details.length == 0) {
const message = `Test case is leaking ${details.length} resource${ return null;
details.length === 1 ? "" : "s" }
}: return { failed: { leakedResources: details } };
- ${details.join("\n - ")}
`;
assert(details.length === 0, message);
}; };
} }
@ -429,9 +407,8 @@ function assertExit(fn, isTest) {
}); });
try { try {
await fn(...new SafeArrayIterator(params)); const innerResult = await fn(...new SafeArrayIterator(params));
} catch (err) { if (innerResult) return innerResult;
throw err;
} finally { } finally {
setExitHandler(null); setExitHandler(null);
} }
@ -441,81 +418,54 @@ function assertExit(fn, isTest) {
function assertTestStepScopes(fn) { function assertTestStepScopes(fn) {
/** @param desc {TestDescription | TestStepDescription} */ /** @param desc {TestDescription | TestStepDescription} */
return async function testStepSanitizer(desc) { return async function testStepSanitizer(desc) {
preValidation(); function getRunningStepDescs() {
// only report waiting after pre-validation const results = [];
if (canStreamReporting(desc) && "parent" in desc) { let childDesc = desc;
stepReportWait(desc); while (childDesc.parent != null) {
const state = MapPrototypeGet(testStates, childDesc.parent.id);
for (const siblingDesc of state.children) {
if (siblingDesc.id == childDesc.id) {
continue;
}
const siblingState = MapPrototypeGet(testStates, siblingDesc.id);
if (!siblingState.completed) {
ArrayPrototypePush(results, siblingDesc);
}
}
childDesc = childDesc.parent;
}
return results;
}
const runningStepDescs = getRunningStepDescs();
const runningStepDescsWithSanitizers = ArrayPrototypeFilter(
runningStepDescs,
(d) => usesSanitizer(d),
);
if (runningStepDescsWithSanitizers.length > 0) {
return {
failed: {
overlapsWithSanitizers: runningStepDescsWithSanitizers.map(
getFullName,
),
},
};
}
if (usesSanitizer(desc) && runningStepDescs.length > 0) {
return {
failed: { hasSanitizersAndOverlaps: runningStepDescs.map(getFullName) },
};
} }
await fn(MapPrototypeGet(testStates, desc.id).context); await fn(MapPrototypeGet(testStates, desc.id).context);
testStepPostValidation(desc); for (const childDesc of MapPrototypeGet(testStates, desc.id).children) {
if (!MapPrototypeGet(testStates, childDesc.id).completed) {
function preValidation() { return { failed: "incompleteSteps" };
const runningStepDescs = getRunningStepDescs();
const runningStepDescsWithSanitizers = ArrayPrototypeFilter(
runningStepDescs,
(d) => usesSanitizer(d),
);
if (runningStepDescsWithSanitizers.length > 0) {
throw new Error(
"Cannot start test step while another test step with sanitizers is running.\n" +
runningStepDescsWithSanitizers
.map((d) => ` * ${getFullName(d)}`)
.join("\n"),
);
}
if (usesSanitizer(desc) && runningStepDescs.length > 0) {
throw new Error(
"Cannot start test step with sanitizers while another test step is running.\n" +
runningStepDescs.map((d) => ` * ${getFullName(d)}`).join("\n"),
);
}
function getRunningStepDescs() {
const results = [];
let childDesc = desc;
while (childDesc.parent != null) {
const state = MapPrototypeGet(testStates, childDesc.parent.id);
for (const siblingDesc of state.children) {
if (siblingDesc.id == childDesc.id) {
continue;
}
const siblingState = MapPrototypeGet(testStates, siblingDesc.id);
if (!siblingState.finalized) {
ArrayPrototypePush(results, siblingDesc);
}
}
childDesc = childDesc.parent;
}
return results;
} }
} }
}; };
} }
function testStepPostValidation(desc) {
// check for any running steps
for (const childDesc of MapPrototypeGet(testStates, desc.id).children) {
if (MapPrototypeGet(testStates, childDesc.id).status == "pending") {
throw new Error(
"There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).",
);
}
}
// check if an ancestor already completed
let currentDesc = desc.parent;
while (currentDesc != null) {
if (MapPrototypeGet(testStates, currentDesc.id).finalized) {
throw new Error(
"Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).",
);
}
currentDesc = currentDesc.parent;
}
}
function pledgePermissions(permissions) { function pledgePermissions(permissions) {
return ops.op_pledge_test_permissions( return ops.op_pledge_test_permissions(
serializePermissions(permissions), serializePermissions(permissions),
@ -573,18 +523,14 @@ function withPermissions(fn, permissions) {
* @typedef {{ * @typedef {{
* context: TestContext, * context: TestContext,
* children: TestStepDescription[], * children: TestStepDescription[],
* finalized: boolean, * completed: boolean,
* }} TestState * }} TestState
* *
* @typedef {{ * @typedef {{
* context: TestContext, * context: TestContext,
* children: TestStepDescription[], * children: TestStepDescription[],
* finalized: boolean, * completed: boolean,
* status: "pending" | "ok" | ""failed" | ignored", * failed: boolean,
* error: unknown,
* elapsed: number | null,
* reportedWait: boolean,
* reportedResult: boolean,
* }} TestStepState * }} TestStepState
* *
* @typedef {{ * @typedef {{
@ -701,13 +647,6 @@ function test(
// Delete this prop in case the user passed it. It's used to detect steps. // Delete this prop in case the user passed it. It's used to detect steps.
delete testDesc.parent; delete testDesc.parent;
testDesc.fn = wrapTestFnWithSanitizers(testDesc.fn, testDesc);
if (testDesc.permissions) {
testDesc.fn = withPermissions(
testDesc.fn,
testDesc.permissions,
);
}
testDesc.origin = getTestOrigin(); testDesc.origin = getTestOrigin();
const jsError = core.destructureError(new Error()); const jsError = core.destructureError(new Error());
testDesc.location = { testDesc.location = {
@ -724,7 +663,7 @@ function test(
MapPrototypeSet(testStates, testDesc.id, { MapPrototypeSet(testStates, testDesc.id, {
context: createTestContext(testDesc), context: createTestContext(testDesc),
children: [], children: [],
finalized: false, completed: false,
}); });
} }
@ -832,28 +771,20 @@ async function runTest(desc) {
if (desc.ignore) { if (desc.ignore) {
return "ignored"; return "ignored";
} }
let testFn = wrapTestFnWithSanitizers(desc.fn, desc);
if (!("parent" in desc) && desc.permissions) {
testFn = withPermissions(
testFn,
desc.permissions,
);
}
try { try {
await desc.fn(desc); const result = await testFn(desc);
const failCount = failedChildStepsCount(desc); if (result) return result;
return failCount === 0 ? "ok" : { const failedSteps = failedChildStepsCount(desc);
"failed": core.destructureError( return failedSteps === 0 ? "ok" : { failed: { failedSteps } };
new Error(
`${failCount} test step${failCount === 1 ? "" : "s"} failed.`,
),
),
};
} catch (error) { } catch (error) {
return { return { failed: { jsError: core.destructureError(error) } };
"failed": core.destructureError(error),
};
} finally {
const state = MapPrototypeGet(testStates, desc.id);
state.finalized = true;
// ensure the children report their result
for (const childDesc of state.children) {
stepReportResult(childDesc);
}
} }
} }
@ -1094,6 +1025,11 @@ async function runTests({
const earlier = DateNow(); const earlier = DateNow();
const result = await runTest(desc); const result = await runTest(desc);
const elapsed = DateNow() - earlier; const elapsed = DateNow() - earlier;
const state = MapPrototypeGet(testStates, desc.id);
state.completed = true;
for (const childDesc of state.children) {
stepReportResult(childDesc, { failed: "incomplete" }, 0);
}
ops.op_dispatch_test_event({ ops.op_dispatch_test_event({
result: [desc.id, result, elapsed], result: [desc.id, result, elapsed],
}); });
@ -1153,7 +1089,7 @@ async function runBenchmarks() {
function getFullName(desc) { function getFullName(desc) {
if ("parent" in desc) { if ("parent" in desc) {
return `${desc.parent.name} > ${desc.name}`; return `${getFullName(desc.parent)} ... ${desc.name}`;
} }
return desc.name; return desc.name;
} }
@ -1162,74 +1098,23 @@ function usesSanitizer(desc) {
return desc.sanitizeResources || desc.sanitizeOps || desc.sanitizeExit; return desc.sanitizeResources || desc.sanitizeOps || desc.sanitizeExit;
} }
function canStreamReporting(desc) { function stepReportResult(desc, result, elapsed) {
let currentDesc = desc;
while (currentDesc != null) {
if (!usesSanitizer(currentDesc)) {
return false;
}
currentDesc = currentDesc.parent;
}
for (const childDesc of MapPrototypeGet(testStates, desc.id).children) {
const state = MapPrototypeGet(testStates, childDesc.id);
if (!usesSanitizer(childDesc) && !state.finalized) {
return false;
}
}
return true;
}
function stepReportWait(desc) {
const state = MapPrototypeGet(testStates, desc.id); const state = MapPrototypeGet(testStates, desc.id);
if (state.reportedWait) {
return;
}
ops.op_dispatch_test_event({ stepWait: desc.id });
state.reportedWait = true;
}
function stepReportResult(desc) {
const state = MapPrototypeGet(testStates, desc.id);
if (state.reportedResult) {
return;
}
stepReportWait(desc);
for (const childDesc of state.children) { for (const childDesc of state.children) {
stepReportResult(childDesc); stepReportResult(childDesc, { failed: "incomplete" }, 0);
}
let result;
if (state.status == "pending" || state.status == "failed") {
result = {
[state.status]: state.error && core.destructureError(state.error),
};
} else {
result = state.status;
} }
ops.op_dispatch_test_event({ ops.op_dispatch_test_event({
stepResult: [desc.id, result, state.elapsed], stepResult: [desc.id, result, elapsed],
}); });
state.reportedResult = true;
} }
function failedChildStepsCount(desc) { function failedChildStepsCount(desc) {
return ArrayPrototypeFilter( return ArrayPrototypeFilter(
MapPrototypeGet(testStates, desc.id).children, MapPrototypeGet(testStates, desc.id).children,
(d) => MapPrototypeGet(testStates, d.id).status === "failed", (d) => MapPrototypeGet(testStates, d.id).failed,
).length; ).length;
} }
/** If a test validation error already occurred then don't bother checking
* the sanitizers as that will create extra noise.
*/
function shouldSkipSanitizers(desc) {
try {
testStepPostValidation(desc);
return false;
} catch {
return true;
}
}
/** @param desc {TestDescription | TestStepDescription} */ /** @param desc {TestDescription | TestStepDescription} */
function createTestContext(desc) { function createTestContext(desc) {
let parent; let parent;
@ -1266,7 +1151,7 @@ function createTestContext(desc) {
* @param maybeFn {((t: TestContext) => void | Promise<void>) | undefined} * @param maybeFn {((t: TestContext) => void | Promise<void>) | undefined}
*/ */
async step(nameOrFnOrOptions, maybeFn) { async step(nameOrFnOrOptions, maybeFn) {
if (MapPrototypeGet(testStates, desc.id).finalized) { if (MapPrototypeGet(testStates, desc.id).completed) {
throw new Error( throw new Error(
"Cannot run test step after parent scope has finished execution. " + "Cannot run test step after parent scope has finished execution. " +
"Ensure any `.step(...)` calls are executed before their parent scope completes execution.", "Ensure any `.step(...)` calls are executed before their parent scope completes execution.",
@ -1322,12 +1207,8 @@ function createTestContext(desc) {
const state = { const state = {
context: createTestContext(stepDesc), context: createTestContext(stepDesc),
children: [], children: [],
finalized: false, failed: false,
status: "pending", completed: false,
error: null,
elapsed: null,
reportedWait: false,
reportedResult: false,
}; };
MapPrototypeSet(testStates, stepDesc.id, state); MapPrototypeSet(testStates, stepDesc.id, state);
ArrayPrototypePush( ArrayPrototypePush(
@ -1335,56 +1216,14 @@ function createTestContext(desc) {
stepDesc, stepDesc,
); );
try { ops.op_dispatch_test_event({ stepWait: stepDesc.id });
if (stepDesc.ignore) { const earlier = DateNow();
state.status = "ignored"; const result = await runTest(stepDesc);
state.finalized = true; const elapsed = DateNow() - earlier;
if (canStreamReporting(stepDesc)) { state.failed = !!result.failed;
stepReportResult(stepDesc); state.completed = true;
} stepReportResult(stepDesc, result, elapsed);
return false; return result == "ok";
}
const testFn = wrapTestFnWithSanitizers(stepDesc.fn, stepDesc);
const start = DateNow();
try {
await testFn(stepDesc);
if (failedChildStepsCount(stepDesc) > 0) {
state.status = "failed";
} else {
state.status = "ok";
}
} catch (error) {
state.error = error;
state.status = "failed";
}
state.elapsed = DateNow() - start;
if (MapPrototypeGet(testStates, stepDesc.parent.id).finalized) {
// always point this test out as one that was still running
// if the parent step finalized
state.status = "pending";
}
state.finalized = true;
if (state.reportedWait && canStreamReporting(stepDesc)) {
stepReportResult(stepDesc);
}
return state.status === "ok";
} finally {
if (canStreamReporting(stepDesc.parent)) {
const parentState = MapPrototypeGet(testStates, stepDesc.parent.id);
// flush any buffered steps
for (const childDesc of parentState.children) {
stepReportResult(childDesc);
}
}
}
}, },
}; };
} }

24
cli/lsp/testing/execution.rs
View file

@ -439,9 +439,6 @@ impl TestRun {
test::TestStepResult::Failed(_) => { test::TestStepResult::Failed(_) => {
summary.failed_steps += 1; summary.failed_steps += 1;
} }
test::TestStepResult::Pending(_) => {
summary.pending_steps += 1;
}
} }
reporter.report_step_result( reporter.report_step_result(
test_steps.get(&id).unwrap(), test_steps.get(&id).unwrap(),
@ -449,6 +446,7 @@ impl TestRun {
duration, duration,
); );
} }
test::TestEvent::Sigint => {}
} }
} }
@ -716,11 +714,10 @@ impl LspTestReporter {
test: desc.into(), test: desc.into(),
}) })
} }
test::TestResult::Failed(js_error) => { test::TestResult::Failed(failure) => {
let err_string = test::format_test_error(js_error);
self.progress(lsp_custom::TestRunProgressMessage::Failed { self.progress(lsp_custom::TestRunProgressMessage::Failed {
test: desc.into(), test: desc.into(),
messages: as_test_messages(err_string, false), messages: as_test_messages(failure.to_string(), false),
duration: Some(elapsed as u32), duration: Some(elapsed as u32),
}) })
} }
@ -830,24 +827,13 @@ impl LspTestReporter {
test: desc.into(), test: desc.into(),
}) })
} }
test::TestStepResult::Failed(js_error) => { test::TestStepResult::Failed(failure) => {
let messages = if let Some(js_error) = js_error {
let err_string = test::format_test_error(js_error);
as_test_messages(err_string, false)
} else {
vec![]
};
self.progress(lsp_custom::TestRunProgressMessage::Failed { self.progress(lsp_custom::TestRunProgressMessage::Failed {
test: desc.into(), test: desc.into(),
messages, messages: as_test_messages(failure.to_string(), false),
duration: Some(elapsed as u32), duration: Some(elapsed as u32),
}) })
} }
test::TestStepResult::Pending(_) => {
self.progress(lsp_custom::TestRunProgressMessage::Enqueued {
test: desc.into(),
})
}
} }
} }

21
cli/tests/integration/test_tests.rs
View file

@ -4,6 +4,7 @@ use deno_core::url::Url;
use test_util as util; use test_util as util;
use util::assert_contains; use util::assert_contains;
use util::env_vars_for_npm_tests; use util::env_vars_for_npm_tests;
use util::wildcard_match;
use util::TestContext; use util::TestContext;
#[test] #[test]
@ -444,6 +445,26 @@ itest!(parallel_output {
exit_code: 1, exit_code: 1,
}); });
#[test]
fn sigint_with_hanging_test() {
util::with_pty(
&[
"test",
"--quiet",
"--no-check",
"test/sigint_with_hanging_test.ts",
],
|mut console| {
std::thread::sleep(std::time::Duration::from_secs(1));
console.write_line("\x03");
wildcard_match(
include_str!("../testdata/test/sigint_with_hanging_test.out"),
&console.read_all_output(),
);
},
);
}
itest!(package_json_basic { itest!(package_json_basic {
args: "test", args: "test",
output: "package_json/basic/lib.test.out", output: "package_json/basic/lib.test.out",

7
cli/tests/testdata/test/ops_sanitizer_missing_details.out vendored
View file

@ -5,12 +5,9 @@ test 1 ... FAILED [WILDCARD]
ERRORS ERRORS
test 1 => ./test/ops_sanitizer_missing_details.ts:[WILDCARD] test 1 => ./test/ops_sanitizer_missing_details.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 1 async operation to op_write was started in this test, but never completed.
- 1 async operation to op_write was started in this test, but never completed.
To get more details where ops were leaked, run again with --trace-ops flag. To get more details where ops were leaked, run again with --trace-ops flag.
at [WILDCARD]
FAILURES FAILURES

14
cli/tests/testdata/test/ops_sanitizer_multiple_timeout_tests.out vendored
View file

@ -6,9 +6,8 @@ test 2 ... FAILED ([WILDCARD])
ERRORS ERRORS
test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD] test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operations were started here:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operations were started here:
at [WILDCARD] at [WILDCARD]
at setTimeout ([WILDCARD]) at setTimeout ([WILDCARD])
at test ([WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:4:3) at test ([WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:4:3)
@ -21,12 +20,9 @@ error: AssertionError: Test case is leaking async ops.
at [WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:8:27 at [WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:8:27
at [WILDCARD] at [WILDCARD]
at [WILDCARD]
test 2 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD] test 2 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operations were started here:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operations were started here:
at [WILDCARD] at [WILDCARD]
at setTimeout ([WILDCARD]) at setTimeout ([WILDCARD])
at test ([WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:4:3) at test ([WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:4:3)
@ -39,8 +35,6 @@ error: AssertionError: Test case is leaking async ops.
at [WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:10:27 at [WILDCARD]/testdata/test/ops_sanitizer_multiple_timeout_tests.ts:10:27
at [WILDCARD] at [WILDCARD]
at [WILDCARD]
FAILURES FAILURES
test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD] test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD]

14
cli/tests/testdata/test/ops_sanitizer_multiple_timeout_tests_no_trace.out vendored
View file

@ -6,20 +6,14 @@ test 2 ... FAILED ([WILDCARD])
ERRORS ERRORS
test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD] test 1 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call.
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call.
To get more details where ops were leaked, run again with --trace-ops flag. To get more details where ops were leaked, run again with --trace-ops flag.
at [WILDCARD]
test 2 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD] test 2 => ./test/ops_sanitizer_multiple_timeout_tests.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call.
- 2 async operations to sleep for a duration were started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call.
To get more details where ops were leaked, run again with --trace-ops flag. To get more details where ops were leaked, run again with --trace-ops flag.
at [WILDCARD]
FAILURES FAILURES

7
cli/tests/testdata/test/ops_sanitizer_unstable.out vendored
View file

@ -6,16 +6,13 @@ leak interval ... FAILED ([WILDCARD])
ERRORS ERRORS
leak interval => ./test/ops_sanitizer_unstable.ts:[WILDCARD] leak interval => ./test/ops_sanitizer_unstable.ts:[WILDCARD]
error: AssertionError: Test case is leaking async ops. error: Leaking async ops:
- 1 async operation to sleep for a duration was started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operation was started here:
- 1 async operation to sleep for a duration was started in this test, but never completed. This is often caused by not cancelling a `setTimeout` or `setInterval` call. The operation was started here:
at [WILDCARD] at [WILDCARD]
at setInterval ([WILDCARD]) at setInterval ([WILDCARD])
at [WILDCARD]/testdata/test/ops_sanitizer_unstable.ts:3:3 at [WILDCARD]/testdata/test/ops_sanitizer_unstable.ts:3:3
at [WILDCARD] at [WILDCARD]
at [WILDCARD]
FAILURES FAILURES
leak interval => ./test/ops_sanitizer_unstable.ts:[WILDCARD] leak interval => ./test/ops_sanitizer_unstable.ts:[WILDCARD]

53
cli/tests/testdata/test/parallel_output.out vendored
View file

@ -12,45 +12,44 @@ Hello, world! (from step 4)
./test/parallel_output.ts => step output ... ok ([WILDCARD]ms) ./test/parallel_output.ts => step output ... ok ([WILDCARD]ms)
./test/parallel_output.ts => step failures ... step 1 ... ok ([WILDCARD]ms) ./test/parallel_output.ts => step failures ... step 1 ... ok ([WILDCARD]ms)
./test/parallel_output.ts => step failures ... step 2 ... FAILED ([WILDCARD]ms) ./test/parallel_output.ts => step failures ... step 2 ... FAILED ([WILDCARD]ms)
error: Error: Fail.
throw new Error("Fail.");
^
at file:///[WILDCARD]/test/parallel_output.ts:15:11
at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:14:11
./test/parallel_output.ts => step failures ... step 3 ... FAILED ([WILDCARD]ms) ./test/parallel_output.ts => step failures ... step 3 ... FAILED ([WILDCARD]ms)
error: Error: Fail. ./test/parallel_output.ts => step failures ... FAILED (due to 2 failed steps) ([WILDCARD]ms)
await t.step("step 3", () => Promise.reject(new Error("Fail.")));
^
at file:///[WILDCARD]/test/parallel_output.ts:17:47
at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:17:11
./test/parallel_output.ts => step failures ... FAILED ([WILDCARD]ms)
./test/parallel_output.ts => step nested failure ... step 1 ... inner 1 ... ok ([WILDCARD]ms) ./test/parallel_output.ts => step nested failure ... step 1 ... inner 1 ... ok ([WILDCARD]ms)
./test/parallel_output.ts => step nested failure ... step 1 ... inner 2 ... FAILED ([WILDCARD]ms) ./test/parallel_output.ts => step nested failure ... step 1 ... inner 2 ... FAILED ([WILDCARD]ms)
error: Error: Failed. ./test/parallel_output.ts => step nested failure ... step 1 ... FAILED (due to 1 failed step) ([WILDCARD]ms)
throw new Error("Failed."); ./test/parallel_output.ts => step nested failure ... FAILED (due to 1 failed step) ([WILDCARD]ms)
^
at file:///[WILDCARD]/test/parallel_output.ts:24:13
at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:23:13
./test/parallel_output.ts => step nested failure ... step 1 ... FAILED ([WILDCARD]ms)
./test/parallel_output.ts => step nested failure ... FAILED ([WILDCARD]ms)
ERRORS ERRORS
step failures => ./test/parallel_output.ts:12:6 step failures ... step 2 => ./test/parallel_output.ts:14:11
error: Error: 2 test steps failed. error: Error: Fail.
throw new Error("Fail.");
^
at file:///[WILDCARD]/test/parallel_output.ts:15:11
at [WILDCARD] at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:14:11
step nested failure => ./test/parallel_output.ts:20:6 step failures ... step 3 => ./test/parallel_output.ts:17:11
error: Error: 1 test step failed. error: Error: Fail.
await t.step("step 3", () => Promise.reject(new Error("Fail.")));
^
at file:///[WILDCARD]/test/parallel_output.ts:17:47
at [WILDCARD] at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:17:11
step nested failure ... step 1 ... inner 2 => ./test/parallel_output.ts:23:13
error: Error: Failed.
throw new Error("Failed.");
^
at file:///[WILDCARD]/test/parallel_output.ts:24:13
at [WILDCARD]
at file:///[WILDCARD]/test/parallel_output.ts:23:13
FAILURES FAILURES
step failures => ./test/parallel_output.ts:12:6 step failures ... step 2 => ./test/parallel_output.ts:14:11
step nested failure => ./test/parallel_output.ts:20:6 step failures ... step 3 => ./test/parallel_output.ts:17:11
step nested failure ... step 1 ... inner 2 => ./test/parallel_output.ts:23:13
FAILED | 1 passed (6 steps) | 2 failed (4 steps) ([WILDCARD]ms) FAILED | 1 passed (6 steps) | 2 failed (4 steps) ([WILDCARD]ms)

9
cli/tests/testdata/test/resource_sanitizer.out vendored
View file

@ -5,12 +5,9 @@ leak ... FAILED ([WILDCARD])
ERRORS ERRORS
leak => ./test/resource_sanitizer.ts:[WILDCARD] leak => ./test/resource_sanitizer.ts:[WILDCARD]
error: AssertionError: Test case is leaking 2 resources: error: Leaking resources:
- The stdin pipe (rid 0) was opened before the test started, but was closed during the test. Do not close resources in a test that were not created during that test.
- The stdin pipe (rid 0) was opened before the test started, but was closed during the test. Do not close resources in a test that were not created during that test. - A file (rid 3) was opened during the test, but not closed during the test. Close the file handle by calling `file.close()`.
- A file (rid 3) was opened during the test, but not closed during the test. Close the file handle by calling `file.close()`.
at [WILDCARD]
FAILURES FAILURES

10
cli/tests/testdata/test/sigint_with_hanging_test.out vendored Normal file
View file

@ -0,0 +1,10 @@
running 1 test from [WILDCARD]/hanging_test.ts
test ...
step 1 ...
step 2 ...
SIGINT The following tests were pending:
test => [WILDCARD]/hanging_test.ts:3:6
test ... step 1 => [WILDCARD]/hanging_test.ts:9:13
test ... step 1 ... step 2 => [WILDCARD]/hanging_test.ts:10:15

15
cli/tests/testdata/test/sigint_with_hanging_test.ts vendored Normal file
View file

@ -0,0 +1,15 @@
setInterval(() => {}, 10000);
Deno.test({
name: "test",
sanitizeOps: false,
sanitizeExit: false,
sanitizeResources: false,
async fn(t) {
await t.step("step 1", async (t) => {
await t.step("step 2", async () => {
await new Promise(() => {});
});
});
},
});

61
cli/tests/testdata/test/steps/failing_steps.out vendored
View file

@ -3,47 +3,42 @@ running 3 tests from ./test/steps/failing_steps.ts
nested failure ... nested failure ...
step 1 ... step 1 ...
inner 1 ... FAILED ([WILDCARD]) inner 1 ... FAILED ([WILDCARD])
error: Error: Failed.
throw new Error("Failed.");
^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
[WILDCARD]
inner 2 ... ok ([WILDCARD]) inner 2 ... ok ([WILDCARD])
step 1 ... FAILED ([WILDCARD]) step 1 ... FAILED (due to 1 failed step) ([WILDCARD])
nested failure ... FAILED ([WILDCARD]) nested failure ... FAILED (due to 1 failed step) ([WILDCARD])
multiple test step failures ... multiple test step failures ...
step 1 ... FAILED ([WILDCARD]) step 1 ... FAILED ([WILDCARD])
error: Error: Fail.
throw new Error("Fail.");
^
[WILDCARD]
step 2 ... FAILED ([WILDCARD]) step 2 ... FAILED ([WILDCARD])
error: Error: Fail. multiple test step failures ... FAILED (due to 2 failed steps) ([WILDCARD])
await t.step("step 2", () => Promise.reject(new Error("Fail.")));
^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
[WILDCARD]
multiple test step failures ... FAILED ([WILDCARD])
failing step in failing test ... failing step in failing test ...
step 1 ... FAILED ([WILDCARD]) step 1 ... FAILED ([WILDCARD])
error: Error: Fail.
throw new Error("Fail.");
^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
at [WILDCARD]
failing step in failing test ... FAILED ([WILDCARD]) failing step in failing test ... FAILED ([WILDCARD])
ERRORS ERRORS
nested failure => ./test/steps/failing_steps.ts:[WILDCARD] nested failure ... step 1 ... inner 1 => ./test/steps/failing_steps.ts:[WILDCARD]
error: Error: 1 test step failed. error: Error: Failed.
at runTest (ext:cli/40_testing.js:[WILDCARD]) throw new Error("Failed.");
at async runTests (ext:cli/40_testing.js:[WILDCARD]) ^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
multiple test step failures => ./test/steps/failing_steps.ts:[WILDCARD] multiple test step failures ... step 1 => ./test/steps/failing_steps.ts:[WILDCARD]
error: Error: 2 test steps failed. error: Error: Fail.
at runTest (ext:cli/40_testing.js:[WILDCARD]) throw new Error("Fail.");
at async runTests (ext:cli/40_testing.js:[WILDCARD]) ^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
multiple test step failures ... step 2 => ./test/steps/failing_steps.ts:[WILDCARD]
error: Error: Fail.
await t.step("step 2", () => Promise.reject(new Error("Fail.")));
^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
failing step in failing test ... step 1 => ./test/steps/failing_steps.ts:[WILDCARD]
error: Error: Fail.
throw new Error("Fail.");
^
at [WILDCARD]/failing_steps.ts:[WILDCARD]
failing step in failing test => ./test/steps/failing_steps.ts:[WILDCARD] failing step in failing test => ./test/steps/failing_steps.ts:[WILDCARD]
error: Error: Fail test. error: Error: Fail test.
@ -53,8 +48,10 @@ error: Error: Fail test.
FAILURES FAILURES
nested failure => ./test/steps/failing_steps.ts:[WILDCARD] nested failure ... step 1 ... inner 1 => ./test/steps/failing_steps.ts:[WILDCARD]
multiple test step failures => ./test/steps/failing_steps.ts:[WILDCARD] multiple test step failures ... step 1 => ./test/steps/failing_steps.ts:[WILDCARD]
multiple test step failures ... step 2 => ./test/steps/failing_steps.ts:[WILDCARD]
failing step in failing test ... step 1 => ./test/steps/failing_steps.ts:[WILDCARD]
failing step in failing test => ./test/steps/failing_steps.ts:[WILDCARD] failing step in failing test => ./test/steps/failing_steps.ts:[WILDCARD]
FAILED | 0 passed (1 step) | 3 failed (5 steps) ([WILDCARD]) FAILED | 0 passed (1 step) | 3 failed (5 steps) ([WILDCARD])

110
cli/tests/testdata/test/steps/invalid_usage.out vendored
View file

@ -4,65 +4,33 @@ capturing ...
some step ... ok ([WILDCARD]) some step ... ok ([WILDCARD])
capturing ... FAILED ([WILDCARD]) capturing ... FAILED ([WILDCARD])
top level missing await ... top level missing await ...
step ... pending ([WILDCARD]) step ... INCOMPLETE
top level missing await ... FAILED ([WILDCARD]) top level missing await ... FAILED (due to incomplete steps) ([WILDCARD])
inner missing await ... inner missing await ...
step ... step ...
inner ... pending ([WILDCARD]) inner ... INCOMPLETE
error: Error: Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). step ... FAILED (due to incomplete steps) ([WILDCARD])
at [WILDCARD] inner missing await ... FAILED (due to 1 failed step) ([WILDCARD])
at async TestContext.step [WILDCARD]
step ... FAILED ([WILDCARD])
error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
await t.step("step", (t) => {
^
at [WILDCARD]
at async fn ([WILDCARD]/invalid_usage.ts:[WILDCARD])
inner missing await ... FAILED ([WILDCARD])
parallel steps with sanitizers ... parallel steps with sanitizers ...
step 1 ... pending ([WILDCARD]) step 1 ... INCOMPLETE
step 2 ... FAILED ([WILDCARD]) step 2 ... FAILED ([WILDCARD])
error: Error: Cannot start test step while another test step with sanitizers is running. parallel steps with sanitizers ... FAILED (due to incomplete steps) ([WILDCARD])
* parallel steps with sanitizers > step 1
await t.step("step 2", () => {});
^
at [WILDCARD]
at [WILDCARD]/invalid_usage.ts:[WILDCARD]
parallel steps with sanitizers ... FAILED ([WILDCARD])
parallel steps when first has sanitizer ... parallel steps when first has sanitizer ...
step 1 ... pending ([WILDCARD]) step 1 ... ok ([WILDCARD])
step 2 ... FAILED ([WILDCARD]) step 2 ... FAILED ([WILDCARD])
error: Error: Cannot start test step while another test step with sanitizers is running. parallel steps when first has sanitizer ... FAILED (due to 1 failed step) ([WILDCARD])
* parallel steps when first has sanitizer > step 1
await t.step({
^
at [WILDCARD]
at [WILDCARD]/invalid_usage.ts:[WILDCARD]
parallel steps when first has sanitizer ... FAILED ([WILDCARD])
parallel steps when second has sanitizer ... parallel steps when second has sanitizer ...
step 1 ... ok ([WILDCARD]) step 1 ... ok ([WILDCARD])
step 2 ... FAILED ([WILDCARD]) step 2 ... FAILED ([WILDCARD])
error: Error: Cannot start test step with sanitizers while another test step is running. parallel steps when second has sanitizer ... FAILED (due to 1 failed step) ([WILDCARD])
* parallel steps when second has sanitizer > step 1
await t.step({
^
at [WILDCARD]
at [WILDCARD]/invalid_usage.ts:[WILDCARD]
parallel steps when second has sanitizer ... FAILED ([WILDCARD])
parallel steps where only inner tests have sanitizers ... parallel steps where only inner tests have sanitizers ...
step 1 ... step 1 ...
step inner ... ok ([WILDCARD]) step inner ... ok ([WILDCARD])
step 1 ... ok ([WILDCARD]) step 1 ... ok ([WILDCARD])
step 2 ... step 2 ...
step inner ... FAILED ([WILDCARD]) step inner ... FAILED ([WILDCARD])
error: Error: Cannot start test step with sanitizers while another test step is running. step 2 ... FAILED (due to 1 failed step) ([WILDCARD])
* parallel steps where only inner tests have sanitizers > step 1 parallel steps where only inner tests have sanitizers ... FAILED (due to 1 failed step) ([WILDCARD])
await t.step({
^
at [WILDCARD]
at [WILDCARD]/invalid_usage.ts:[WILDCARD]
step 2 ... FAILED ([WILDCARD])
parallel steps where only inner tests have sanitizers ... FAILED ([WILDCARD])
ERRORS ERRORS
@ -73,38 +41,42 @@ error: Error: Cannot run test step after parent scope has finished execution. En
at TestContext.step ([WILDCARD]) at TestContext.step ([WILDCARD])
at [WILDCARD]/invalid_usage.ts:[WILDCARD] at [WILDCARD]/invalid_usage.ts:[WILDCARD]
top level missing await => ./test/steps/invalid_usage.ts:[WILDCARD] top level missing await ... step => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). error: Didn't complete before parent. Await step with `await t.step(...)`.
[WILDCARD]
inner missing await => ./test/steps/invalid_usage.ts:[WILDCARD] inner missing await ... step ... inner => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Error: 1 test step failed. error: Didn't complete before parent. Await step with `await t.step(...)`.
at [WILDCARD]
parallel steps with sanitizers => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps with sanitizers ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). error: Started test step while another test step with sanitizers was running:
[WILDCARD] * parallel steps with sanitizers ... step 1
parallel steps when first has sanitizer => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps with sanitizers ... step 1 => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Error: 1 test step failed. error: Didn't complete before parent. Await step with `await t.step(...)`.
at runTest ([WILDCARD])
at [WILDCARD]
parallel steps when second has sanitizer => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps when first has sanitizer ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Error: 1 test step failed. error: Started test step while another test step with sanitizers was running:
at runTest ([WILDCARD]) * parallel steps when first has sanitizer ... step 1
at [WILDCARD]
parallel steps when second has sanitizer ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Started test step with sanitizers while another test step was running:
* parallel steps when second has sanitizer ... step 1
parallel steps where only inner tests have sanitizers ... step 2 ... step inner => ./test/steps/invalid_usage.ts:[WILDCARD]
error: Started test step with sanitizers while another test step was running:
* parallel steps where only inner tests have sanitizers ... step 1
FAILURES FAILURES
capturing => ./test/steps/invalid_usage.ts:[WILDCARD] capturing => ./test/steps/invalid_usage.ts:3:6
top level missing await => ./test/steps/invalid_usage.ts:[WILDCARD] top level missing await ... step => ./test/steps/invalid_usage.ts:[WILDCARD]
inner missing await => ./test/steps/invalid_usage.ts:[WILDCARD] inner missing await ... step ... inner => ./test/steps/invalid_usage.ts:[WILDCARD]
parallel steps with sanitizers => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps with sanitizers ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
parallel steps when first has sanitizer => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps with sanitizers ... step 1 => ./test/steps/invalid_usage.ts:[WILDCARD]
parallel steps when second has sanitizer => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps when first has sanitizer ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
parallel steps where only inner tests have sanitizers => ./test/steps/invalid_usage.ts:[WILDCARD] parallel steps when second has sanitizer ... step 2 => ./test/steps/invalid_usage.ts:[WILDCARD]
parallel steps where only inner tests have sanitizers ... step 2 ... step inner => ./test/steps/invalid_usage.ts:[WILDCARD]
FAILED | 0 passed (4 steps) | 7 failed (10 steps) ([WILDCARD]) FAILED | 0 passed (5 steps) | 7 failed (9 steps) ([WILDCARD])
error: Test failed error: Test failed

2
cli/tests/testdata/test/steps/invalid_usage.ts vendored
View file

@ -11,7 +11,7 @@ Deno.test("capturing", async (t) => {
Deno.test("top level missing await", (t) => { Deno.test("top level missing await", (t) => {
t.step("step", () => { t.step("step", () => {
return new Promise((resolve) => setTimeout(resolve, 10)); return new Promise(() => {});
}); });
}); });

84
cli/tests/unit_node/process_test.ts
View file

@ -155,24 +155,29 @@ Deno.test({
name: "process.on signal", name: "process.on signal",
ignore: Deno.build.os == "windows", ignore: Deno.build.os == "windows",
async fn() { async fn() {
const promise = deferred(); const process = new Deno.Command(Deno.execPath(), {
let c = 0; args: [
const listener = () => { "eval",
c += 1; `
}; import process from "node:process";
process.on("SIGINT", listener); setInterval(() => {}, 1000);
setTimeout(async () => { process.on("SIGINT", () => {
// Sends SIGINT 3 times. console.log("foo");
for (const _ of Array(3)) { });
await delay(20); `,
Deno.kill(Deno.pid, "SIGINT"); ],
} stdout: "piped",
stderr: "null",
}).spawn();
await delay(500);
for (const _ of Array(3)) {
process.kill("SIGINT");
await delay(20); await delay(20);
Deno.removeSignalListener("SIGINT", listener); }
promise.resolve(); await delay(20);
}); process.kill("SIGTERM");
await promise; const output = await process.output();
assertEquals(c, 3); assertEquals(new TextDecoder().decode(output.stdout), "foo\nfoo\nfoo\n");
}, },
}); });
@ -180,24 +185,35 @@ Deno.test({
name: "process.off signal", name: "process.off signal",
ignore: Deno.build.os == "windows", ignore: Deno.build.os == "windows",
async fn() { async fn() {
const promise = deferred(); const process = new Deno.Command(Deno.execPath(), {
let c = 0; args: [
const listener = () => { "eval",
c += 1; `
process.off("SIGINT", listener); import process from "node:process";
}; setInterval(() => {}, 1000);
process.on("SIGINT", listener); const listener = () => {
setTimeout(async () => { console.log("foo");
// Sends SIGINT 3 times. process.off("SIGINT")
for (const _ of Array(3)) { };
await delay(20); process.on("SIGINT", listener);
Deno.kill(Deno.pid, "SIGINT"); `,
} ],
stdout: "piped",
stderr: "null",
}).spawn();
await delay(500);
for (const _ of Array(3)) {
try {
process.kill("SIGINT");
} catch { /* should die after the first one */ }
await delay(20); await delay(20);
promise.resolve(); }
}); await delay(20);
await promise; try {
assertEquals(c, 1); process.kill("SIGTERM");
} catch { /* should be dead, avoid hanging just in case */ }
const output = await process.output();
assertEquals(new TextDecoder().decode(output.stdout), "foo\n");
}, },
}); });

461
cli/tools/test.rs
View file

@ -39,6 +39,7 @@ use deno_runtime::permissions::Permissions;
use deno_runtime::permissions::PermissionsContainer; use deno_runtime::permissions::PermissionsContainer;
use deno_runtime::tokio_util::run_local; use deno_runtime::tokio_util::run_local;
use indexmap::IndexMap; use indexmap::IndexMap;
use indexmap::IndexSet;
use log::Level; use log::Level;
use rand::rngs::SmallRng; use rand::rngs::SmallRng;
use rand::seq::SliceRandom; use rand::seq::SliceRandom;
@ -47,6 +48,8 @@ use regex::Regex;
use serde::Deserialize; use serde::Deserialize;
use std::cell::RefCell; use std::cell::RefCell;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
use std::collections::HashSet; use std::collections::HashSet;
use std::fmt::Write as _; use std::fmt::Write as _;
use std::io::Read; use std::io::Read;
@ -58,8 +61,10 @@ use std::sync::atomic::AtomicUsize;
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use std::time::Instant; use std::time::Instant;
use tokio::signal;
use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::unbounded_channel;
use tokio::sync::mpsc::UnboundedSender; use tokio::sync::mpsc::UnboundedSender;
use tokio::sync::mpsc::WeakUnboundedSender;
/// The test mode is used to determine how a specifier is to be tested. /// The test mode is used to determine how a specifier is to be tested.
#[derive(Debug, Clone, Eq, PartialEq)] #[derive(Debug, Clone, Eq, PartialEq)]
@ -154,13 +159,100 @@ pub enum TestOutput {
Bytes(Vec<u8>), Bytes(Vec<u8>),
} }
#[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Debug, Clone, PartialEq, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum TestFailure {
JsError(Box<JsError>),
FailedSteps(usize),
IncompleteSteps,
LeakedOps(Vec<String>, bool), // Details, isOpCallTracingEnabled
LeakedResources(Vec<String>), // Details
// The rest are for steps only.
Incomplete,
OverlapsWithSanitizers(IndexSet<String>), // Long names of overlapped tests
HasSanitizersAndOverlaps(IndexSet<String>), // Long names of overlapped tests
}
impl ToString for TestFailure {
fn to_string(&self) -> String {
match self {
TestFailure::JsError(js_error) => format_test_error(js_error),
TestFailure::FailedSteps(1) => "1 test step failed.".to_string(),
TestFailure::FailedSteps(n) => format!("{} test steps failed.", n),
TestFailure::IncompleteSteps => "Completed while steps were still running. Ensure all steps are awaited with `await t.step(...)`.".to_string(),
TestFailure::Incomplete => "Didn't complete before parent. Await step with `await t.step(...)`.".to_string(),
TestFailure::LeakedOps(details, is_op_call_tracing_enabled) => {
let mut string = "Leaking async ops:".to_string();
for detail in details {
string.push_str(&format!("\n - {}", detail));
}
if !is_op_call_tracing_enabled {
string.push_str("\nTo get more details where ops were leaked, run again with --trace-ops flag.");
}
string
}
TestFailure::LeakedResources(details) => {
let mut string = "Leaking resources:".to_string();
for detail in details {
string.push_str(&format!("\n - {}", detail));
}
string
}
TestFailure::OverlapsWithSanitizers(long_names) => {
let mut string = "Started test step while another test step with sanitizers was running:".to_string();
for long_name in long_names {
string.push_str(&format!("\n * {}", long_name));
}
string
}
TestFailure::HasSanitizersAndOverlaps(long_names) => {
let mut string = "Started test step with sanitizers while another test step was running:".to_string();
for long_name in long_names {
string.push_str(&format!("\n * {}", long_name));
}
string
}
}
}
}
impl TestFailure {
fn format_label(&self) -> String {
match self {
TestFailure::Incomplete => colors::gray("INCOMPLETE").to_string(),
_ => colors::red("FAILED").to_string(),
}
}
fn format_inline_summary(&self) -> Option<String> {
match self {
TestFailure::FailedSteps(1) => Some("due to 1 failed step".to_string()),
TestFailure::FailedSteps(n) => Some(format!("due to {} failed steps", n)),
TestFailure::IncompleteSteps => {
Some("due to incomplete steps".to_string())
}
_ => None,
}
}
fn hide_in_summary(&self) -> bool {
// These failure variants are hidden in summaries because they are caused
// by child errors that will be summarized separately.
matches!(
self,
TestFailure::FailedSteps(_) | TestFailure::IncompleteSteps
)
}
}
#[allow(clippy::derive_partial_eq_without_eq)] #[allow(clippy::derive_partial_eq_without_eq)]
#[derive(Debug, Clone, PartialEq, Deserialize)] #[derive(Debug, Clone, PartialEq, Deserialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
pub enum TestResult { pub enum TestResult {
Ok, Ok,
Ignored, Ignored,
Failed(Box<JsError>), Failed(TestFailure),
Cancelled, Cancelled,
} }
@ -193,18 +285,7 @@ impl TestStepDescription {
pub enum TestStepResult { pub enum TestStepResult {
Ok, Ok,
Ignored, Ignored,
Failed(Option<Box<JsError>>), Failed(TestFailure),
Pending(Option<Box<JsError>>),
}
impl TestStepResult {
fn error(&self) -> Option<&JsError> {
match self {
TestStepResult::Failed(Some(error)) => Some(error),
TestStepResult::Pending(Some(error)) => Some(error),
_ => None,
}
}
} }
#[derive(Debug, Clone, Eq, PartialEq, Deserialize)] #[derive(Debug, Clone, Eq, PartialEq, Deserialize)]
@ -228,6 +309,7 @@ pub enum TestEvent {
StepRegister(TestStepDescription), StepRegister(TestStepDescription),
StepWait(usize), StepWait(usize),
StepResult(usize, TestStepResult, u64), StepResult(usize, TestStepResult, u64),
Sigint,
} }
#[derive(Debug, Clone, Deserialize)] #[derive(Debug, Clone, Deserialize)]
@ -238,11 +320,10 @@ pub struct TestSummary {
pub ignored: usize, pub ignored: usize,
pub passed_steps: usize, pub passed_steps: usize,
pub failed_steps: usize, pub failed_steps: usize,
pub pending_steps: usize,
pub ignored_steps: usize, pub ignored_steps: usize,
pub filtered_out: usize, pub filtered_out: usize,
pub measured: usize, pub measured: usize,
pub failures: Vec<(TestDescription, Box<JsError>)>, pub failures: Vec<(TestDescription, TestFailure)>,
pub uncaught_errors: Vec<(String, Box<JsError>)>, pub uncaught_errors: Vec<(String, Box<JsError>)>,
} }
@ -262,7 +343,6 @@ impl TestSummary {
ignored: 0, ignored: 0,
passed_steps: 0, passed_steps: 0,
failed_steps: 0, failed_steps: 0,
pending_steps: 0,
ignored_steps: 0, ignored_steps: 0,
filtered_out: 0, filtered_out: 0,
measured: 0, measured: 0,
@ -280,10 +360,12 @@ struct PrettyTestReporter {
parallel: bool, parallel: bool,
echo_output: bool, echo_output: bool,
in_new_line: bool, in_new_line: bool,
last_wait_id: Option<usize>, scope_test_id: Option<usize>,
cwd: Url, cwd: Url,
did_have_user_output: bool, did_have_user_output: bool,
started_tests: bool, started_tests: bool,
child_results_buffer:
HashMap<usize, IndexMap<usize, (TestStepDescription, TestStepResult, u64)>>,
} }
impl PrettyTestReporter { impl PrettyTestReporter {
@ -292,10 +374,11 @@ impl PrettyTestReporter {
parallel, parallel,
echo_output, echo_output,
in_new_line: true, in_new_line: true,
last_wait_id: None, scope_test_id: None,
cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(), cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(),
did_have_user_output: false, did_have_user_output: false,
started_tests: false, started_tests: false,
child_results_buffer: Default::default(),
} }
} }
@ -316,7 +399,7 @@ impl PrettyTestReporter {
self.in_new_line = false; self.in_new_line = false;
// flush for faster feedback when line buffered // flush for faster feedback when line buffered
std::io::stdout().flush().unwrap(); std::io::stdout().flush().unwrap();
self.last_wait_id = Some(description.id); self.scope_test_id = Some(description.id);
} }
fn to_relative_path_or_remote_url(&self, path_or_url: &str) -> String { fn to_relative_path_or_remote_url(&self, path_or_url: &str) -> String {
@ -341,7 +424,7 @@ impl PrettyTestReporter {
self.in_new_line = false; self.in_new_line = false;
// flush for faster feedback when line buffered // flush for faster feedback when line buffered
std::io::stdout().flush().unwrap(); std::io::stdout().flush().unwrap();
self.last_wait_id = Some(description.id); self.scope_test_id = Some(description.id);
} }
fn force_report_step_result( fn force_report_step_result(
@ -350,32 +433,53 @@ impl PrettyTestReporter {
result: &TestStepResult, result: &TestStepResult,
elapsed: u64, elapsed: u64,
) { ) {
let status = match result {
TestStepResult::Ok => colors::green("ok").to_string(),
TestStepResult::Ignored => colors::yellow("ignored").to_string(),
TestStepResult::Pending(_) => colors::gray("pending").to_string(),
TestStepResult::Failed(_) => colors::red("FAILED").to_string(),
};
self.write_output_end(); self.write_output_end();
if self.in_new_line || self.last_wait_id != Some(description.id) { if self.in_new_line || self.scope_test_id != Some(description.id) {
self.force_report_step_wait(description); self.force_report_step_wait(description);
} }
println!( if !self.parallel {
" {} {}", let child_results = self
status, .child_results_buffer
colors::gray(format!("({})", display::human_elapsed(elapsed.into()))) .remove(&description.id)
); .unwrap_or_default();
for (desc, result, elapsed) in child_results.values() {
if let Some(js_error) = result.error() { self.force_report_step_result(desc, result, *elapsed);
let err_string = format_test_error(js_error); }
let err_string = format!("{}: {}", colors::red_bold("error"), err_string); if !child_results.is_empty() {
for line in err_string.lines() { self.force_report_step_wait(description);
println!("{}{}", " ".repeat(description.level + 1), line);
} }
} }
let status = match &result {
TestStepResult::Ok => colors::green("ok").to_string(),
TestStepResult::Ignored => colors::yellow("ignored").to_string(),
TestStepResult::Failed(failure) => failure.format_label(),
};
print!(" {}", status);
if let TestStepResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
print!(" ({})", inline_summary)
}
}
if !matches!(result, TestStepResult::Failed(TestFailure::Incomplete)) {
print!(
" {}",
colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
);
}
println!();
self.in_new_line = true; self.in_new_line = true;
if self.parallel {
self.scope_test_id = None;
} else {
self.scope_test_id = Some(description.parent_id);
}
self
.child_results_buffer
.entry(description.parent_id)
.or_default()
.remove(&description.id);
} }
fn write_output_end(&mut self) { fn write_output_end(&mut self) {
@ -442,23 +546,28 @@ impl PrettyTestReporter {
} }
self.write_output_end(); self.write_output_end();
if self.in_new_line || self.last_wait_id != Some(description.id) { if self.in_new_line || self.scope_test_id != Some(description.id) {
self.force_report_wait(description); self.force_report_wait(description);
} }
let status = match result { let status = match result {
TestResult::Ok => colors::green("ok").to_string(), TestResult::Ok => colors::green("ok").to_string(),
TestResult::Ignored => colors::yellow("ignored").to_string(), TestResult::Ignored => colors::yellow("ignored").to_string(),
TestResult::Failed(_) => colors::red("FAILED").to_string(), TestResult::Failed(failure) => failure.format_label(),
TestResult::Cancelled => colors::gray("cancelled").to_string(), TestResult::Cancelled => colors::gray("cancelled").to_string(),
}; };
print!(" {}", status);
if let TestResult::Failed(failure) = result {
if let Some(inline_summary) = failure.format_inline_summary() {
print!(" ({})", inline_summary)
}
}
println!( println!(
" {} {}", " {}",
status,
colors::gray(format!("({})", display::human_elapsed(elapsed.into()))) colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
); );
self.in_new_line = true; self.in_new_line = true;
self.scope_test_id = None;
} }
fn report_uncaught_error(&mut self, origin: &str, _error: &JsError) { fn report_uncaught_error(&mut self, origin: &str, _error: &JsError) {
@ -477,14 +586,14 @@ impl PrettyTestReporter {
fn report_step_register(&mut self, _description: &TestStepDescription) {} fn report_step_register(&mut self, _description: &TestStepDescription) {}
fn report_step_wait(&mut self, description: &TestStepDescription) { fn report_step_wait(&mut self, description: &TestStepDescription) {
if !self.parallel { if !self.parallel && self.scope_test_id == Some(description.parent_id) {
self.force_report_step_wait(description); self.force_report_step_wait(description);
} }
} }
fn report_step_result( fn report_step_result(
&mut self, &mut self,
description: &TestStepDescription, desc: &TestStepDescription,
result: &TestStepResult, result: &TestStepResult,
elapsed: u64, elapsed: u64,
tests: &IndexMap<usize, TestDescription>, tests: &IndexMap<usize, TestDescription>,
@ -492,35 +601,34 @@ impl PrettyTestReporter {
) { ) {
if self.parallel { if self.parallel {
self.write_output_end(); self.write_output_end();
let root;
let mut ancestor_names = vec![];
let mut current_desc = description;
loop {
if let Some(step_desc) = test_steps.get(&current_desc.parent_id) {
ancestor_names.push(&step_desc.name);
current_desc = step_desc;
} else {
root = tests.get(&current_desc.parent_id).unwrap();
break;
}
}
ancestor_names.reverse();
print!( print!(
"{}", "{} {} ...",
colors::gray(format!( colors::gray(format!(
"{} =>", "{} =>",
self.to_relative_path_or_remote_url(&description.origin) self.to_relative_path_or_remote_url(&desc.origin)
)) )),
self.format_test_step_ancestry(desc, tests, test_steps)
); );
print!(" {} ...", root.name);
for name in ancestor_names {
print!(" {name} ...");
}
print!(" {} ...", description.name);
self.in_new_line = false; self.in_new_line = false;
self.last_wait_id = Some(description.id); self.scope_test_id = Some(desc.id);
self.force_report_step_result(desc, result, elapsed);
} else {
let sibling_results =
self.child_results_buffer.entry(desc.parent_id).or_default();
if self.scope_test_id == Some(desc.id)
|| self.scope_test_id == Some(desc.parent_id)
{
let sibling_results = std::mem::take(sibling_results);
self.force_report_step_result(desc, result, elapsed);
// Flush buffered sibling results.
for (desc, result, elapsed) in sibling_results.values() {
self.force_report_step_result(desc, result, *elapsed);
}
} else {
sibling_results
.insert(desc.id, (desc.clone(), result.clone(), elapsed));
}
} }
self.force_report_step_result(description, result, elapsed);
} }
fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration) { fn report_summary(&mut self, summary: &TestSummary, elapsed: &Duration) {
@ -528,14 +636,14 @@ impl PrettyTestReporter {
#[allow(clippy::type_complexity)] // Type alias doesn't look better here #[allow(clippy::type_complexity)] // Type alias doesn't look better here
let mut failures_by_origin: BTreeMap< let mut failures_by_origin: BTreeMap<
String, String,
(Vec<(&TestDescription, &JsError)>, Option<&JsError>), (Vec<(&TestDescription, &TestFailure)>, Option<&JsError>),
> = BTreeMap::default(); > = BTreeMap::default();
let mut failure_titles = vec![]; let mut failure_titles = vec![];
for (description, js_error) in &summary.failures { for (description, failure) in &summary.failures {
let (failures, _) = failures_by_origin let (failures, _) = failures_by_origin
.entry(description.origin.clone()) .entry(description.origin.clone())
.or_default(); .or_default();
failures.push((description, js_error.as_ref())); failures.push((description, failure));
} }
for (origin, js_error) in &summary.uncaught_errors { for (origin, js_error) in &summary.uncaught_errors {
let (_, uncaught_error) = let (_, uncaught_error) =
@ -544,27 +652,14 @@ impl PrettyTestReporter {
} }
println!("\n{}\n", colors::white_bold_on_red(" ERRORS ")); println!("\n{}\n", colors::white_bold_on_red(" ERRORS "));
for (origin, (failures, uncaught_error)) in failures_by_origin { for (origin, (failures, uncaught_error)) in failures_by_origin {
for (description, js_error) in failures { for (description, failure) in failures {
let failure_title = format!( if !failure.hide_in_summary() {
"{} {}", let failure_title = self.format_test_for_summary(description);
&description.name, println!("{}", &failure_title);
colors::gray(format!( println!("{}: {}", colors::red_bold("error"), failure.to_string());
"=> {}:{}:{}", println!();
self.to_relative_path_or_remote_url( failure_titles.push(failure_title);
&description.location.file_name }
),
description.location.line_number,
description.location.column_number
))
);
println!("{}", &failure_title);
println!(
"{}: {}",
colors::red_bold("error"),
format_test_error(js_error)
);
println!();
failure_titles.push(failure_title);
} }
if let Some(js_error) = uncaught_error { if let Some(js_error) = uncaught_error {
let failure_title = format!( let failure_title = format!(
@ -613,7 +708,7 @@ impl PrettyTestReporter {
summary.passed, summary.passed,
get_steps_text(summary.passed_steps), get_steps_text(summary.passed_steps),
summary.failed, summary.failed,
get_steps_text(summary.failed_steps + summary.pending_steps), get_steps_text(summary.failed_steps),
) )
.unwrap(); .unwrap();
@ -647,6 +742,98 @@ impl PrettyTestReporter {
); );
self.in_new_line = true; self.in_new_line = true;
} }
fn report_sigint(
&mut self,
tests_pending: &HashSet<usize>,
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) {
if tests_pending.is_empty() {
return;
}
let mut formatted_pending = BTreeSet::new();
for id in tests_pending {
if let Some(desc) = tests.get(id) {
formatted_pending.insert(self.format_test_for_summary(desc));
}
if let Some(desc) = test_steps.get(id) {
formatted_pending
.insert(self.format_test_step_for_summary(desc, tests, test_steps));
}
}
println!(
"\n{} The following tests were pending:\n",
colors::intense_blue("SIGINT")
);
for entry in formatted_pending {
println!("{}", entry);
}
println!();
self.in_new_line = true;
}
fn format_test_step_ancestry(
&self,
desc: &TestStepDescription,
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) -> String {
let root;
let mut ancestor_names = vec![];
let mut current_desc = desc;
loop {
if let Some(step_desc) = test_steps.get(&current_desc.parent_id) {
ancestor_names.push(&step_desc.name);
current_desc = step_desc;
} else {
root = tests.get(&current_desc.parent_id).unwrap();
break;
}
}
ancestor_names.reverse();
let mut result = String::new();
result.push_str(&root.name);
result.push_str(" ... ");
for name in ancestor_names {
result.push_str(name);
result.push_str(" ... ");
}
result.push_str(&desc.name);
result
}
fn format_test_for_summary(&self, desc: &TestDescription) -> String {
format!(
"{} {}",
&desc.name,
colors::gray(format!(
"=> {}:{}:{}",
self.to_relative_path_or_remote_url(&desc.location.file_name),
desc.location.line_number,
desc.location.column_number
))
)
}
fn format_test_step_for_summary(
&self,
desc: &TestStepDescription,
tests: &IndexMap<usize, TestDescription>,
test_steps: &IndexMap<usize, TestStepDescription>,
) -> String {
let long_name = self.format_test_step_ancestry(desc, tests, test_steps);
format!(
"{} {}",
long_name,
colors::gray(format!(
"=> {}:{}:{}",
self.to_relative_path_or_remote_url(&desc.location.file_name),
desc.location.line_number,
desc.location.column_number
))
)
}
} }
fn abbreviate_test_error(js_error: &JsError) -> JsError { fn abbreviate_test_error(js_error: &JsError) -> JsError {
@ -1007,6 +1194,12 @@ async fn test_specifiers(
let sender = TestEventSender::new(sender); let sender = TestEventSender::new(sender);
let concurrent_jobs = options.concurrent_jobs; let concurrent_jobs = options.concurrent_jobs;
let sender_ = sender.downgrade();
let sigint_handler_handle = tokio::task::spawn(async move {
signal::ctrl_c().await.unwrap();
sender_.upgrade().map(|s| s.send(TestEvent::Sigint).ok());
});
let join_handles = let join_handles =
specifiers_with_mode specifiers_with_mode
.into_iter() .into_iter()
@ -1060,6 +1253,7 @@ async fn test_specifiers(
let earlier = Instant::now(); let earlier = Instant::now();
let mut tests = IndexMap::new(); let mut tests = IndexMap::new();
let mut test_steps = IndexMap::new(); let mut test_steps = IndexMap::new();
let mut tests_started = HashSet::new();
let mut tests_with_result = HashSet::new(); let mut tests_with_result = HashSet::new();
let mut summary = TestSummary::new(); let mut summary = TestSummary::new();
let mut used_only = false; let mut used_only = false;
@ -1083,7 +1277,9 @@ async fn test_specifiers(
} }
TestEvent::Wait(id) => { TestEvent::Wait(id) => {
reporter.report_wait(tests.get(&id).unwrap()); if tests_started.insert(id) {
reporter.report_wait(tests.get(&id).unwrap());
}
} }
TestEvent::Output(output) => { TestEvent::Output(output) => {
@ -1100,9 +1296,11 @@ async fn test_specifiers(
TestResult::Ignored => { TestResult::Ignored => {
summary.ignored += 1; summary.ignored += 1;
} }
TestResult::Failed(error) => { TestResult::Failed(failure) => {
summary.failed += 1; summary.failed += 1;
summary.failures.push((description.clone(), error.clone())); summary
.failures
.push((description.clone(), failure.clone()));
} }
TestResult::Cancelled => { TestResult::Cancelled => {
unreachable!("should be handled in TestEvent::UncaughtError"); unreachable!("should be handled in TestEvent::UncaughtError");
@ -1130,36 +1328,65 @@ async fn test_specifiers(
} }
TestEvent::StepWait(id) => { TestEvent::StepWait(id) => {
reporter.report_step_wait(test_steps.get(&id).unwrap()); if tests_started.insert(id) {
reporter.report_step_wait(test_steps.get(&id).unwrap());
}
} }
TestEvent::StepResult(id, result, duration) => { TestEvent::StepResult(id, result, duration) => {
match &result { if tests_with_result.insert(id) {
TestStepResult::Ok => { let description = test_steps.get(&id).unwrap();
summary.passed_steps += 1; match &result {
TestStepResult::Ok => {
summary.passed_steps += 1;
}
TestStepResult::Ignored => {
summary.ignored_steps += 1;
}
TestStepResult::Failed(failure) => {
summary.failed_steps += 1;
summary.failures.push((
TestDescription {
id: description.id,
name: reporter.format_test_step_ancestry(
description,
&tests,
&test_steps,
),
origin: description.origin.clone(),
location: description.location.clone(),
},
failure.clone(),
))
}
} }
TestStepResult::Ignored => {
summary.ignored_steps += 1;
}
TestStepResult::Failed(_) => {
summary.failed_steps += 1;
}
TestStepResult::Pending(_) => {
summary.pending_steps += 1;
}
}
reporter.report_step_result( reporter.report_step_result(
test_steps.get(&id).unwrap(), description,
&result, &result,
duration, duration,
&tests,
&test_steps,
);
}
}
TestEvent::Sigint => {
reporter.report_sigint(
&tests_started
.difference(&tests_with_result)
.copied()
.collect(),
&tests, &tests,
&test_steps, &test_steps,
); );
std::process::exit(130);
} }
} }
} }
sigint_handler_handle.abort();
let elapsed = Instant::now().duration_since(earlier); let elapsed = Instant::now().duration_since(earlier);
reporter.report_summary(&summary, &elapsed); reporter.report_summary(&summary, &elapsed);
@ -1598,6 +1825,10 @@ impl TestEventSender {
Ok(()) Ok(())
} }
fn downgrade(&self) -> WeakUnboundedSender<TestEvent> {
self.sender.downgrade()
}
fn flush_stdout_and_stderr(&mut self) -> Result<(), AnyError> { fn flush_stdout_and_stderr(&mut self) -> Result<(), AnyError> {
self.stdout_writer.flush()?; self.stdout_writer.flush()?;
self.stderr_writer.flush()?; self.stderr_writer.flush()?;