From a4b7d563c48d6f386d3383ce35a1319a6834eaf0 Mon Sep 17 00:00:00 2001 From: Matt Mastracci Date: Wed, 13 Sep 2023 16:09:43 -0600 Subject: [PATCH] chore(test_util): ensure that extra expectations are an error even without a filter (#20483) Running `tools/wpt.ts` with a filter would cause an error if there were extra, leftover expectations in expectations.json. These errors would not appear if no filter was passed, often leaving the filtered version of the test runner broken. This also introduces a smarter bit of logic where filters can be specified with a leading slash (`tools/wpt.ts run -- /url` is equivalent to `tools/wpt.ts run -- url`) --- tools/wpt.ts | 53 +++++++++++++++++++++----------- tools/wpt/expectation.json | 62 ++------------------------------------ 2 files changed, 37 insertions(+), 78 deletions(-) diff --git a/tools/wpt.ts b/tools/wpt.ts index 29d3e3517e..8c9d0a447b 100755 --- a/tools/wpt.ts +++ b/tools/wpt.ts @@ -37,6 +37,31 @@ import { blue, bold, green, red, yellow } from "../test_util/std/fmt/colors.ts"; import { writeAll, writeAllSync } from "../test_util/std/streams/write_all.ts"; import { saveExpectation } from "./wpt/utils.ts"; +class TestFilter { + filter?: string[]; + constructor(filter?: string[]) { + this.filter = filter; + } + + matches(path: string): boolean { + if (this.filter === undefined || this.filter.length == 0) { + return true; + } + for (const filter of this.filter) { + if (filter.startsWith("/")) { + if (path.startsWith(filter)) { + return true; + } + } else { + if (path.substring(1).startsWith(filter)) { + return true; + } + } + } + return false; + } +} + const command = Deno.args[0]; switch (command) { @@ -161,11 +186,12 @@ async function run() { const startTime = new Date().getTime(); assert(Array.isArray(rest), "filter must be array"); const expectation = getExpectation(); + const filter = new TestFilter(rest); const tests = discoverTestsToRun( - rest.length == 0 ? undefined : rest, + filter, expectation, ); - assertAllExpectationsHaveTests(expectation, tests, rest); + assertAllExpectationsHaveTests(expectation, tests, filter); const cores = navigator.hardwareConcurrency; console.log(`Going to run ${tests.length} test files on ${cores} cores.`); @@ -295,20 +321,14 @@ async function generateWptReport( function assertAllExpectationsHaveTests( expectation: Expectation, testsToRun: TestToRun[], - filter?: string[], + filter: TestFilter, ): void { const tests = new Set(testsToRun.map((t) => t.path)); const missingTests: string[] = []; - function walk(parentExpectation: Expectation, parent: string) { for (const [key, expectation] of Object.entries(parentExpectation)) { const path = `${parent}/${key}`; - if ( - filter && - !filter.find((filter) => path.substring(1).startsWith(filter)) - ) { - continue; - } + if (!filter.matches(path)) continue; if (typeof expectation == "boolean" || Array.isArray(expectation)) { if (!tests.has(path)) { missingTests.push(path); @@ -336,7 +356,8 @@ function assertAllExpectationsHaveTests( async function update() { assert(Array.isArray(rest), "filter must be array"); const startTime = new Date().getTime(); - const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true); + const filter = new TestFilter(rest); + const tests = discoverTestsToRun(filter, true); console.log(`Going to run ${tests.length} test files.`); const results = await runWithTestUtil(false, async () => { @@ -668,7 +689,7 @@ function createReportTestCase(expectation: boolean | string[]) { } function discoverTestsToRun( - filter?: string[], + filter: TestFilter, expectation: Expectation | string[] | boolean = getExpectation(), ): TestToRun[] { const manifestFolder = getManifest().items.testharness; @@ -736,12 +757,8 @@ function discoverTestsToRun( ); } - if ( - filter && - !filter.find((filter) => finalPath.substring(1).startsWith(filter)) - ) { - continue; - } + if (!filter.matches(finalPath)) continue; + testsToRun.push({ path: finalPath, url, diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json index f76e7266cb..6e0eb2e1a2 100644 --- a/tools/wpt/expectation.json +++ b/tools/wpt/expectation.json @@ -2186,14 +2186,6 @@ "importVectorKeys step: EdDSA Ed448 verifying with wrong algorithm name" ] }, - "wrapKey_unwrapKey": { - "wrapKey_unwrapKey.https.any.worker.html": { - "ignore": true - }, - "wrapKey_unwrapKey.https.any.html": { - "ignore": true - } - }, "algorithm-discards-context.https.window.html": false }, "console": { @@ -2214,9 +2206,7 @@ "AbortSignal.any.html": true, "AbortSignal.any.worker.html": true, "event.any.html": true, - "event.any.worker.html": true, - "abort-signal-any.tentative.any.html": false, - "abort-signal-any.tentative.any.worker.html": false + "event.any.worker.html": true }, "events": { "AddEventListenerOptions-once.any.html": true, @@ -2455,7 +2445,6 @@ "general.any.worker.html": true, "non-transferable-buffers.any.html": true, "non-transferable-buffers.any.worker.html": true, - "enqueue-with-detached-buffer.window.html": false, "tee.any.html": true, "tee.any.worker.html": true, "respond-after-enqueue.any.html": true, @@ -2719,8 +2708,6 @@ "is.tentative.any.worker.html": true, "toString.tentative.any.html": true, "toString.tentative.any.worker.html": true, - "type.tentative.any.html": false, - "type.tentative.any.worker.html": false, "identity.tentative.any.html": true, "identity.tentative.any.worker.html": true }, @@ -2817,9 +2804,6 @@ "global-immutable-prototype.any.html": [ "Setting to a different prototype" ], - "global-immutable-prototype.any.worker.html": { - "ignore": true - }, "global-object-implicit-this-value.any.html": [ "Global object's getter throws when called on incompatible object", "Global object's setter throws when called on incompatible object", @@ -4423,9 +4407,6 @@ "accept-header.any.worker.html": true, "conditional-get.any.html": false, "conditional-get.any.worker.html": false, - "error-after-response.any.html": { - "ignore": true - }, "header-value-combining.any.html": false, "header-value-combining.any.worker.html": false, "header-value-null-byte.any.html": true, @@ -4434,25 +4415,6 @@ "historical.any.worker.html": true, "http-response-code.any.html": true, "http-response-code.any.worker.html": true, - "request-upload.any.html": [ - "Fetch with POST with ReadableStream containing String", - "Fetch with POST with ReadableStream containing null", - "Fetch with POST with ReadableStream containing number", - "Fetch with POST with ReadableStream containing ArrayBuffer", - "Fetch with POST with ReadableStream containing Blob", - "Fetch with POST with text body on 421 response should be retried once on new connection." - ], - "request-upload.any.worker.html": [ - "Fetch with POST with ReadableStream containing String", - "Fetch with POST with ReadableStream containing null", - "Fetch with POST with ReadableStream containing number", - "Fetch with POST with ReadableStream containing ArrayBuffer", - "Fetch with POST with ReadableStream containing Blob", - "Fetch with POST with text body on 421 response should be retried once on new connection." - ], - "response-null-body.any.worker.html": { - "ignore": true - }, "response-url.sub.any.html": true, "response-url.sub.any.worker.html": true, "scheme-about.any.html": true, @@ -4492,12 +4454,6 @@ "error-after-response.any.worker.html": false, "keepalive.any.html": false, "mediasource.window.html": false, - "mode-no-cors.sub.any.html": { - "ignore": true - }, - "mode-no-cors.sub.any.worker.html": { - "ignore": true - }, "mode-same-origin.any.html": [ "Fetch https://web-platform.test:8443/fetch/api/resources/top.txt with same-origin mode", "Fetch http://www1.web-platform.test:8000/fetch/api/resources/top.txt with same-origin mode", @@ -4645,10 +4601,7 @@ "header X-METHOD-OVERRIDE is forbidden to use value trace," ], "request-referrer.any.html": false, - "request-referrer.any.worker.html": false, - "response-null-body.any.html": { - "ignore": true - } + "request-referrer.any.worker.html": false }, "response": { "json.any.html": true, @@ -5844,7 +5797,6 @@ "preflight-cache.https.window.html": false, "redirect.https.window.html": false, "service-worker-background-fetch.https.window.html": false, - "service-worker-fetch.https.window.html": false, "service-worker-update.https.window.html": false, "service-worker.https.window.html": false, "shared-worker-blob-fetch.https.window.html": false, @@ -6534,7 +6486,6 @@ "service-worker-coep-credentialless-proxy.https.window.html": false, "service-worker-coep-none-proxy.https.window.html": false, "service-worker.https.window.html": false, - "shared-worker.https.window.html": false, "video.https.window.html": false }, "cross-origin-isolated-permission-iframe.https.window.html": false, @@ -7479,12 +7430,6 @@ "constructor.any.html?wss": true, "constructor.any.worker.html?wpt_flags=h2": false, "constructor.any.worker.html?wss": true, - "abort.any.html?wpt_flags=h2": { - "ignore": true - }, - "abort.any.worker.html?wpt_flags=h2": { - "ignore": true - }, "backpressure-receive.any.worker.html?wpt_flags=h2": false } }, @@ -7566,9 +7511,6 @@ "unexpected-self-properties.worker.html": true } }, - "dedicated-worker-from-blob-url.window.html": { - "ignore": true - }, "dedicated-worker-in-data-url-context.window.html": [ "Create a dedicated worker in a data url frame", "Create a dedicated worker in a data url dedicated worker",