mirror of
https://github.com/denoland/deno.git
synced 2024-12-22 07:14:47 -05:00
chore(test_util): ensure that extra expectations are an error even without a filter (#20483)
Running `tools/wpt.ts` with a filter would cause an error if there were extra, leftover expectations in expectations.json. These errors would not appear if no filter was passed, often leaving the filtered version of the test runner broken. This also introduces a smarter bit of logic where filters can be specified with a leading slash (`tools/wpt.ts run -- /url` is equivalent to `tools/wpt.ts run -- url`)
This commit is contained in:
Matt Mastracci
GitHub
parent
81d50e1b66
commit
a4b7d563c4
2 changed files with 37 additions and 78 deletions
53
tools/wpt.ts
53
tools/wpt.ts
|
|
@ -37,6 +37,31 @@ import { blue, bold, green, red, yellow } from "../test_util/std/fmt/colors.ts";
|
|||
import { writeAll, writeAllSync } from "../test_util/std/streams/write_all.ts";
|
||||
import { saveExpectation } from "./wpt/utils.ts";
|
||||
|
||||
class TestFilter {
|
||||
filter?: string[];
|
||||
constructor(filter?: string[]) {
|
||||
this.filter = filter;
|
||||
}
|
||||
|
||||
matches(path: string): boolean {
|
||||
if (this.filter === undefined || this.filter.length == 0) {
|
||||
return true;
|
||||
}
|
||||
for (const filter of this.filter) {
|
||||
if (filter.startsWith("/")) {
|
||||
if (path.startsWith(filter)) {
|
||||
return true;
|
||||
}
|
||||
} else {
|
||||
if (path.substring(1).startsWith(filter)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const command = Deno.args[0];
|
||||
|
||||
switch (command) {
|
||||
|
|
@ -161,11 +186,12 @@ async function run() {
|
|||
const startTime = new Date().getTime();
|
||||
assert(Array.isArray(rest), "filter must be array");
|
||||
const expectation = getExpectation();
|
||||
const filter = new TestFilter(rest);
|
||||
const tests = discoverTestsToRun(
|
||||
rest.length == 0 ? undefined : rest,
|
||||
filter,
|
||||
expectation,
|
||||
);
|
||||
assertAllExpectationsHaveTests(expectation, tests, rest);
|
||||
assertAllExpectationsHaveTests(expectation, tests, filter);
|
||||
const cores = navigator.hardwareConcurrency;
|
||||
console.log(`Going to run ${tests.length} test files on ${cores} cores.`);
|
||||
|
||||
|
|
@ -295,20 +321,14 @@ async function generateWptReport(
|
|||
function assertAllExpectationsHaveTests(
|
||||
expectation: Expectation,
|
||||
testsToRun: TestToRun[],
|
||||
filter?: string[],
|
||||
filter: TestFilter,
|
||||
): void {
|
||||
const tests = new Set(testsToRun.map((t) => t.path));
|
||||
const missingTests: string[] = [];
|
||||
|
||||
function walk(parentExpectation: Expectation, parent: string) {
|
||||
for (const [key, expectation] of Object.entries(parentExpectation)) {
|
||||
const path = `${parent}/${key}`;
|
||||
if (
|
||||
filter &&
|
||||
!filter.find((filter) => path.substring(1).startsWith(filter))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (!filter.matches(path)) continue;
|
||||
if (typeof expectation == "boolean" || Array.isArray(expectation)) {
|
||||
if (!tests.has(path)) {
|
||||
missingTests.push(path);
|
||||
|
|
@ -336,7 +356,8 @@ function assertAllExpectationsHaveTests(
|
|||
async function update() {
|
||||
assert(Array.isArray(rest), "filter must be array");
|
||||
const startTime = new Date().getTime();
|
||||
const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
|
||||
const filter = new TestFilter(rest);
|
||||
const tests = discoverTestsToRun(filter, true);
|
||||
console.log(`Going to run ${tests.length} test files.`);
|
||||
|
||||
const results = await runWithTestUtil(false, async () => {
|
||||
|
|
@ -668,7 +689,7 @@ function createReportTestCase(expectation: boolean | string[]) {
|
|||
}
|
||||
|
||||
function discoverTestsToRun(
|
||||
filter?: string[],
|
||||
filter: TestFilter,
|
||||
expectation: Expectation | string[] | boolean = getExpectation(),
|
||||
): TestToRun[] {
|
||||
const manifestFolder = getManifest().items.testharness;
|
||||
|
|
@ -736,12 +757,8 @@ function discoverTestsToRun(
|
|||
);
|
||||
}
|
||||
|
||||
if (
|
||||
filter &&
|
||||
!filter.find((filter) => finalPath.substring(1).startsWith(filter))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (!filter.matches(finalPath)) continue;
|
||||
|
||||
testsToRun.push({
|
||||
path: finalPath,
|
||||
url,
|
||||
|
|
|
|||
|
|
@ -2186,14 +2186,6 @@
|
|||
"importVectorKeys step: EdDSA Ed448 verifying with wrong algorithm name"
|
||||
]
|
||||
},
|
||||
"wrapKey_unwrapKey": {
|
||||
"wrapKey_unwrapKey.https.any.worker.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"wrapKey_unwrapKey.https.any.html": {
|
||||
"ignore": true
|
||||
}
|
||||
},
|
||||
"algorithm-discards-context.https.window.html": false
|
||||
},
|
||||
"console": {
|
||||
|
|
@ -2214,9 +2206,7 @@
|
|||
"AbortSignal.any.html": true,
|
||||
"AbortSignal.any.worker.html": true,
|
||||
"event.any.html": true,
|
||||
"event.any.worker.html": true,
|
||||
"abort-signal-any.tentative.any.html": false,
|
||||
"abort-signal-any.tentative.any.worker.html": false
|
||||
"event.any.worker.html": true
|
||||
},
|
||||
"events": {
|
||||
"AddEventListenerOptions-once.any.html": true,
|
||||
|
|
@ -2455,7 +2445,6 @@
|
|||
"general.any.worker.html": true,
|
||||
"non-transferable-buffers.any.html": true,
|
||||
"non-transferable-buffers.any.worker.html": true,
|
||||
"enqueue-with-detached-buffer.window.html": false,
|
||||
"tee.any.html": true,
|
||||
"tee.any.worker.html": true,
|
||||
"respond-after-enqueue.any.html": true,
|
||||
|
|
@ -2719,8 +2708,6 @@
|
|||
"is.tentative.any.worker.html": true,
|
||||
"toString.tentative.any.html": true,
|
||||
"toString.tentative.any.worker.html": true,
|
||||
"type.tentative.any.html": false,
|
||||
"type.tentative.any.worker.html": false,
|
||||
"identity.tentative.any.html": true,
|
||||
"identity.tentative.any.worker.html": true
|
||||
},
|
||||
|
|
@ -2817,9 +2804,6 @@
|
|||
"global-immutable-prototype.any.html": [
|
||||
"Setting to a different prototype"
|
||||
],
|
||||
"global-immutable-prototype.any.worker.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"global-object-implicit-this-value.any.html": [
|
||||
"Global object's getter throws when called on incompatible object",
|
||||
"Global object's setter throws when called on incompatible object",
|
||||
|
|
@ -4423,9 +4407,6 @@
|
|||
"accept-header.any.worker.html": true,
|
||||
"conditional-get.any.html": false,
|
||||
"conditional-get.any.worker.html": false,
|
||||
"error-after-response.any.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"header-value-combining.any.html": false,
|
||||
"header-value-combining.any.worker.html": false,
|
||||
"header-value-null-byte.any.html": true,
|
||||
|
|
@ -4434,25 +4415,6 @@
|
|||
"historical.any.worker.html": true,
|
||||
"http-response-code.any.html": true,
|
||||
"http-response-code.any.worker.html": true,
|
||||
"request-upload.any.html": [
|
||||
"Fetch with POST with ReadableStream containing String",
|
||||
"Fetch with POST with ReadableStream containing null",
|
||||
"Fetch with POST with ReadableStream containing number",
|
||||
"Fetch with POST with ReadableStream containing ArrayBuffer",
|
||||
"Fetch with POST with ReadableStream containing Blob",
|
||||
"Fetch with POST with text body on 421 response should be retried once on new connection."
|
||||
],
|
||||
"request-upload.any.worker.html": [
|
||||
"Fetch with POST with ReadableStream containing String",
|
||||
"Fetch with POST with ReadableStream containing null",
|
||||
"Fetch with POST with ReadableStream containing number",
|
||||
"Fetch with POST with ReadableStream containing ArrayBuffer",
|
||||
"Fetch with POST with ReadableStream containing Blob",
|
||||
"Fetch with POST with text body on 421 response should be retried once on new connection."
|
||||
],
|
||||
"response-null-body.any.worker.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"response-url.sub.any.html": true,
|
||||
"response-url.sub.any.worker.html": true,
|
||||
"scheme-about.any.html": true,
|
||||
|
|
@ -4492,12 +4454,6 @@
|
|||
"error-after-response.any.worker.html": false,
|
||||
"keepalive.any.html": false,
|
||||
"mediasource.window.html": false,
|
||||
"mode-no-cors.sub.any.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"mode-no-cors.sub.any.worker.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"mode-same-origin.any.html": [
|
||||
"Fetch https://web-platform.test:8443/fetch/api/resources/top.txt with same-origin mode",
|
||||
"Fetch http://www1.web-platform.test:8000/fetch/api/resources/top.txt with same-origin mode",
|
||||
|
|
@ -4645,10 +4601,7 @@
|
|||
"header X-METHOD-OVERRIDE is forbidden to use value trace,"
|
||||
],
|
||||
"request-referrer.any.html": false,
|
||||
"request-referrer.any.worker.html": false,
|
||||
"response-null-body.any.html": {
|
||||
"ignore": true
|
||||
}
|
||||
"request-referrer.any.worker.html": false
|
||||
},
|
||||
"response": {
|
||||
"json.any.html": true,
|
||||
|
|
@ -5844,7 +5797,6 @@
|
|||
"preflight-cache.https.window.html": false,
|
||||
"redirect.https.window.html": false,
|
||||
"service-worker-background-fetch.https.window.html": false,
|
||||
"service-worker-fetch.https.window.html": false,
|
||||
"service-worker-update.https.window.html": false,
|
||||
"service-worker.https.window.html": false,
|
||||
"shared-worker-blob-fetch.https.window.html": false,
|
||||
|
|
@ -6534,7 +6486,6 @@
|
|||
"service-worker-coep-credentialless-proxy.https.window.html": false,
|
||||
"service-worker-coep-none-proxy.https.window.html": false,
|
||||
"service-worker.https.window.html": false,
|
||||
"shared-worker.https.window.html": false,
|
||||
"video.https.window.html": false
|
||||
},
|
||||
"cross-origin-isolated-permission-iframe.https.window.html": false,
|
||||
|
|
@ -7479,12 +7430,6 @@
|
|||
"constructor.any.html?wss": true,
|
||||
"constructor.any.worker.html?wpt_flags=h2": false,
|
||||
"constructor.any.worker.html?wss": true,
|
||||
"abort.any.html?wpt_flags=h2": {
|
||||
"ignore": true
|
||||
},
|
||||
"abort.any.worker.html?wpt_flags=h2": {
|
||||
"ignore": true
|
||||
},
|
||||
"backpressure-receive.any.worker.html?wpt_flags=h2": false
|
||||
}
|
||||
},
|
||||
|
|
@ -7566,9 +7511,6 @@
|
|||
"unexpected-self-properties.worker.html": true
|
||||
}
|
||||
},
|
||||
"dedicated-worker-from-blob-url.window.html": {
|
||||
"ignore": true
|
||||
},
|
||||
"dedicated-worker-in-data-url-context.window.html": [
|
||||
"Create a dedicated worker in a data url frame",
|
||||
"Create a dedicated worker in a data url dedicated worker",
|
||||
|
|
|
|||
Loading…
Reference in a new issue