diff --git a/.github/workflows/wpt_epoch.yml b/.github/workflows/wpt_epoch.yml index 1bc7653218..9fdadf1bf0 100644 --- a/.github/workflows/wpt_epoch.yml +++ b/.github/workflows/wpt_epoch.yml @@ -72,7 +72,7 @@ jobs: deno run --unstable --allow-write --allow-read --allow-net \ --allow-env --allow-run --lock=tools/deno.lock.json \ ./tools/wpt.ts run \ - --binary=$(which deno) --quiet --release --json=wpt.json --wptreport=wptreport.json || true + --binary=$(which deno) --quiet --release --no-ignore --json=wpt.json --wptreport=wptreport.json || true - name: Upload wpt results to wpt.fyi env: diff --git a/tools/wpt.ts b/tools/wpt.ts index 5d0c2e7627..2b61682be7 100755 --- a/tools/wpt.ts +++ b/tools/wpt.ts @@ -25,6 +25,7 @@ import { ManifestFolder, ManifestTestOptions, ManifestTestVariation, + noIgnore, quiet, rest, runPy, @@ -173,6 +174,9 @@ async function run() { test.options, inParallel ? () => {} : createReportTestCase(test.expectation), inspectBrk, + Deno.env.get("CI") + ? { long: 4 * 60_000, default: 4 * 60_000 } + : { long: 60_000, default: 10_000 }, ); results.push({ test, result }); if (inParallel) { @@ -332,6 +336,7 @@ async function update() { test.options, json ? () => {} : createReportTestCase(test.expectation), inspectBrk, + { long: 60_000, default: 10_000 }, ); results.push({ test, result }); reportVariation(result, test.expectation); @@ -367,7 +372,7 @@ async function update() { const currentExpectation = getExpectation(); - for (const result of Object.values(resultTests)) { + for (const [path, result] of Object.entries(resultTests)) { const { passed, failed, testSucceeded } = result; let finalExpectation: boolean | string[]; if (failed.length == 0 && testSucceeded) { @@ -699,7 +704,7 @@ function discoverTestsToRun( typeof expectation.ignore === "boolean", "test entry's `ignore` key must be a boolean", ); - if (expectation.ignore === true) continue; + if (expectation.ignore === true && !noIgnore) continue; } } diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json index ebec7caf79..e9ee8e05e7 100644 --- a/tools/wpt/expectation.json +++ b/tools/wpt/expectation.json @@ -1087,8 +1087,8 @@ "EventTarget-constructible.any.html": true, "EventTarget-constructible.any.worker.html": true, "Event-constructors.any.html": [ - "Untitled 3", - "Untitled 4" + "Event constructors 3", + "Event constructors 4" ], "Event-constructors.any.worker.html": [ "Event constructors 3", diff --git a/tools/wpt/runner.ts b/tools/wpt/runner.ts index a3c22006ae..7e15216be4 100644 --- a/tools/wpt/runner.ts +++ b/tools/wpt/runner.ts @@ -75,13 +75,23 @@ export async function runSingleTest( _options: ManifestTestOptions, reporter: (result: TestCaseResult) => void, inspectBrk: boolean, + timeouts: { long: number; default: number }, ): Promise { + const timeout = _options.timeout === "long" + ? timeouts.long + : timeouts.default; + const filename = url.pathname.substring( + url.pathname.lastIndexOf("/") + 1, + url.pathname.indexOf("."), + ); + const { title } = Object.fromEntries(_options.script_metadata || []); const bundle = await generateBundle(url); const tempFile = await Deno.makeTempFile({ prefix: "wpt-bundle-", suffix: ".js", }); + let interval; try { await Deno.writeTextFile(tempFile, bundle); @@ -107,6 +117,7 @@ export async function runSingleTest( "[]", ); + const start = performance.now(); const proc = new Deno.Command(denoBinary(), { args, env: { @@ -124,10 +135,19 @@ export async function runSingleTest( const lines = proc.stderr.pipeThrough(new TextDecoderStream()).pipeThrough( new TextLineStream(), ); + interval = setInterval(() => { + const passedTime = performance.now() - start; + if (passedTime > timeout) { + proc.kill("SIGINT"); + } + }, 1000); for await (const line of lines) { if (line.startsWith("{")) { const data = JSON.parse(line); const result = { ...data, passed: data.status == 0 }; + if (/^Untitled( \d+)?$/.test(result.name)) { + result.name = `${title || filename}${result.name.slice(8)}`; + } cases.push(result); reporter(result); } else if (line.startsWith("#$#$#{")) { @@ -149,6 +169,7 @@ export async function runSingleTest( stderr, }; } finally { + clearInterval(interval); await Deno.remove(tempFile); } } diff --git a/tools/wpt/utils.ts b/tools/wpt/utils.ts index 47cb8c5ec3..1752dab041 100644 --- a/tools/wpt/utils.ts +++ b/tools/wpt/utils.ts @@ -13,10 +13,11 @@ export const { ["--"]: rest, ["auto-config"]: autoConfig, ["inspect-brk"]: inspectBrk, + ["no-ignore"]: noIgnore, binary, } = parse(Deno.args, { "--": true, - boolean: ["quiet", "release", "no-interactive", "inspect-brk"], + boolean: ["quiet", "release", "no-interactive", "inspect-brk", "no-ignore"], string: ["json", "wptreport", "binary"], });