test(wpt): implement process timeout, fix expectations update, and more... (#17892)

- relands #17872
- updates the timeouts to be re-configurable just for CI
- fixes `./tools/wpt.ts update`
- adds option not "ignore" during, applied to wpt epoch runs only

.github/workflows/wpt_epoch.yml

@@ -72,7 +72,7 @@ jobs:
           deno run --unstable --allow-write --allow-read --allow-net           \
             --allow-env --allow-run --lock=tools/deno.lock.json                \
             ./tools/wpt.ts run                                                 \
-            --binary=$(which deno) --quiet --release --json=wpt.json --wptreport=wptreport.json || true
+            --binary=$(which deno) --quiet --release --no-ignore --json=wpt.json --wptreport=wptreport.json || true
 
       - name: Upload wpt results to wpt.fyi
         env:

tools/wpt.ts

@@ -25,6 +25,7 @@ import {
   ManifestFolder,
   ManifestTestOptions,
   ManifestTestVariation,
+  noIgnore,
   quiet,
   rest,
   runPy,
@@ -173,6 +174,9 @@ async function run() {
           test.options,
           inParallel ? () => {} : createReportTestCase(test.expectation),
           inspectBrk,
+          Deno.env.get("CI")
+            ? { long: 4 * 60_000, default: 4 * 60_000 }
+            : { long: 60_000, default: 10_000 },
         );
         results.push({ test, result });
         if (inParallel) {
@@ -332,6 +336,7 @@ async function update() {
         test.options,
         json ? () => {} : createReportTestCase(test.expectation),
         inspectBrk,
+        { long: 60_000, default: 10_000 },
       );
       results.push({ test, result });
       reportVariation(result, test.expectation);
@@ -367,7 +372,7 @@ async function update() {
 
   const currentExpectation = getExpectation();
 
-  for (const result of Object.values(resultTests)) {
+  for (const [path, result] of Object.entries(resultTests)) {
     const { passed, failed, testSucceeded } = result;
     let finalExpectation: boolean | string[];
     if (failed.length == 0 && testSucceeded) {
@@ -699,7 +704,7 @@ function discoverTestsToRun(
                 typeof expectation.ignore === "boolean",
                 "test entry's `ignore` key must be a boolean",
               );
-              if (expectation.ignore === true) continue;
+              if (expectation.ignore === true && !noIgnore) continue;
             }
           }
 

tools/wpt/expectation.json

@@ -1087,8 +1087,8 @@
       "EventTarget-constructible.any.html": true,
       "EventTarget-constructible.any.worker.html": true,
       "Event-constructors.any.html": [
-        "Untitled 3",
+        "Event constructors 3",
-        "Untitled 4"
+        "Event constructors 4"
       ],
       "Event-constructors.any.worker.html": [
         "Event constructors 3",

tools/wpt/runner.ts

@@ -75,13 +75,23 @@ export async function runSingleTest(
   _options: ManifestTestOptions,
   reporter: (result: TestCaseResult) => void,
   inspectBrk: boolean,
+  timeouts: { long: number; default: number },
 ): Promise<TestResult> {
+  const timeout = _options.timeout === "long"
+    ? timeouts.long
+    : timeouts.default;
+  const filename = url.pathname.substring(
+    url.pathname.lastIndexOf("/") + 1,
+    url.pathname.indexOf("."),
+  );
+  const { title } = Object.fromEntries(_options.script_metadata || []);
   const bundle = await generateBundle(url);
   const tempFile = await Deno.makeTempFile({
     prefix: "wpt-bundle-",
     suffix: ".js",
   });
 
+  let interval;
   try {
     await Deno.writeTextFile(tempFile, bundle);
 
@@ -107,6 +117,7 @@ export async function runSingleTest(
       "[]",
     );
 
+    const start = performance.now();
     const proc = new Deno.Command(denoBinary(), {
       args,
       env: {
@@ -124,10 +135,19 @@ export async function runSingleTest(
     const lines = proc.stderr.pipeThrough(new TextDecoderStream()).pipeThrough(
       new TextLineStream(),
     );
+    interval = setInterval(() => {
+      const passedTime = performance.now() - start;
+      if (passedTime > timeout) {
+        proc.kill("SIGINT");
+      }
+    }, 1000);
     for await (const line of lines) {
       if (line.startsWith("{")) {
         const data = JSON.parse(line);
         const result = { ...data, passed: data.status == 0 };
+        if (/^Untitled( \d+)?$/.test(result.name)) {
+          result.name = `${title || filename}${result.name.slice(8)}`;
+        }
         cases.push(result);
         reporter(result);
       } else if (line.startsWith("#$#$#{")) {
@@ -149,6 +169,7 @@ export async function runSingleTest(
       stderr,
     };
   } finally {
+    clearInterval(interval);
     await Deno.remove(tempFile);
   }
 }

tools/wpt/utils.ts

@@ -13,10 +13,11 @@ export const {
   ["--"]: rest,
   ["auto-config"]: autoConfig,
   ["inspect-brk"]: inspectBrk,
+  ["no-ignore"]: noIgnore,
   binary,
 } = parse(Deno.args, {
   "--": true,
-  boolean: ["quiet", "release", "no-interactive", "inspect-brk"],
+  boolean: ["quiet", "release", "no-interactive", "inspect-brk", "no-ignore"],
   string: ["json", "wptreport", "binary"],
 });