tests: generate and upload wptreport.json (#10869)

These reports can be consumed by tools like `wptreport` or
https://wpt.fyi. The old style report could be removed in a future PR
when wpt.deno.land is updated.

.github/workflows/ci.yml

@@ -177,8 +177,7 @@ jobs:
             ~/.cargo/registry/index
             ~/.cargo/registry/cache
             ~/.cargo/git/db
-          key:
+          key: a-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
-            a-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
 
       # In main branch, always creates fresh cache
       - name: Cache build output (main)
@@ -308,7 +307,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' && matrix.profile == 'release'
         run: |
           deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts setup
-          deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts run --quiet --release --json=wpt.json
+          deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts run --quiet --release --json=wpt.json --wptreport=wptreport.json
 
       - name: Upload wpt results to dl.deno.land
         if: |
@@ -319,6 +318,7 @@ jobs:
           github.ref == 'refs/heads/main'
         run: |
           gsutil cp ./wpt.json gs://dl.deno.land/wpt/$(git rev-parse HEAD).json
+          gsutil cp ./wptreport.json gs://dl.deno.land/wpt/$(git rev-parse HEAD)-wptreport.json
           echo $(git rev-parse HEAD) > wpt-latest.txt
           gsutil cp wpt-latest.txt gs://dl.deno.land/wpt-latest.txt
 
@@ -407,4 +407,3 @@ jobs:
           rm -rf target/*/examples/
           rm -rf target/*/gn_out/
           rm -rf target/*/*.zip
-

tools/wpt.ts

@@ -15,6 +15,7 @@ import {
   cargoBuild,
   checkPy3Available,
   Expectation,
+  generateRunInfo,
   getExpectation,
   getExpectFailForCase,
   getManifest,
@@ -26,6 +27,7 @@ import {
   rest,
   runPy,
   updateManifest,
+  wptreport,
 } from "./wpt/utils.ts";
 import {
   blue,
@@ -148,6 +150,7 @@ interface TestToRun {
 }
 
 async function run() {
+  const startTime = new Date().getTime();
   assert(Array.isArray(rest), "filter must be array");
   const expectation = getExpectation();
   const tests = discoverTestsToRun(
@@ -173,6 +176,7 @@ async function run() {
 
     return results;
   });
+  const endTime = new Date().getTime();
 
   if (json) {
     const minifiedResults = [];
@@ -191,10 +195,66 @@ async function run() {
     }
     await Deno.writeTextFile(json, JSON.stringify(minifiedResults));
   }
+
+  if (wptreport) {
+    const report = await generateWptreport(results, startTime, endTime);
+    await Deno.writeTextFile(wptreport, JSON.stringify(report));
+  }
+
   const code = reportFinal(results);
   Deno.exit(code);
 }
 
+async function generateWptreport(
+  results: { test: TestToRun; result: TestResult }[],
+  startTime: number,
+  endTime: number,
+) {
+  const runInfo = await generateRunInfo();
+  const reportResults = [];
+  for (const { test, result } of results) {
+    const status = result.status !== 0
+      ? "CRASH"
+      : result.harnessStatus?.status === 0
+      ? "OK"
+      : "ERROR";
+    const reportResult = {
+      test: test.url.pathname + test.url.search + test.url.hash,
+      subtests: result.cases.map((case_) => {
+        let expected = undefined;
+        if (!case_.passed) {
+          if (typeof test.expectation === "boolean") {
+            expected = test.expectation ? "PASS" : "FAIL";
+          } else {
+            expected = test.expectation.includes(case_.name) ? "FAIL" : "PASS";
+          }
+        }
+
+        return {
+          name: case_.name,
+          status: case_.passed ? "PASS" : "FAIL",
+          message: case_.message,
+          expected,
+          known_intermittent: [],
+        };
+      }),
+      status,
+      message: result.harnessStatus?.message ??
+        (result.stderr.trim() || null),
+      duration: result.duration,
+      expected: status === "OK" ? undefined : "OK",
+      "known_intermittent": [],
+    };
+    reportResults.push(reportResult);
+  }
+  return {
+    "run_info": runInfo,
+    "time_start": startTime,
+    "time_end": endTime,
+    "results": reportResults,
+  };
+}
+
 // Check that all expectations in the expectations file have a test that will be
 // run.
 function assertAllExpectationsHaveTests(

tools/wpt/runner.ts

@@ -47,10 +47,18 @@ export async function runWithTestUtil<T>(
 
 export interface TestResult {
   cases: TestCaseResult[];
+  harnessStatus: TestHarnessStatus | null;
+  duration: number;
   status: number;
   stderr: string;
 }
 
+export interface TestHarnessStatus {
+  status: number;
+  message: string | null;
+  stack: string | null;
+}
+
 export interface TestCaseResult {
   name: string;
   passed: boolean;
@@ -71,6 +79,8 @@ export async function runSingleTest(
   });
   await Deno.writeTextFile(tempFile, bundle);
 
+  const startTime = new Date().getTime();
+
   const proc = Deno.run({
     cmd: [
       join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`),
@@ -94,6 +104,8 @@ export async function runSingleTest(
   const cases = [];
   let stderr = "";
 
+  let harnessStatus = null;
+
   const lines = readLines(proc.stderr);
   for await (const line of lines) {
     if (line.startsWith("{")) {
@@ -101,15 +113,21 @@ export async function runSingleTest(
       const result = { ...data, passed: data.status == 0 };
       cases.push(result);
       reporter(result);
+    } else if (line.startsWith("#$#$#{")) {
+      harnessStatus = JSON.parse(line.slice(5));
     } else {
       stderr += line + "\n";
       console.error(stderr);
     }
   }
 
+  const duration = new Date().getTime() - startTime;
+
   const { code } = await proc.status();
   return {
     status: code,
+    harnessStatus,
+    duration,
     cases,
     stderr,
   };

tools/wpt/testharnessreport.js

@@ -10,6 +10,13 @@ window.add_result_callback(({ message, name, stack, status }) => {
   }
 });
 
-window.add_completion_callback((_tests, _harnessStatus) => {
+window.add_completion_callback((_tests, harnessStatus) => {
+  const data = new TextEncoder().encode(
+    `#$#$#${JSON.stringify(harnessStatus)}\n`,
+  );
+  let bytesWritten = 0;
+  while (bytesWritten < data.byteLength) {
+    bytesWritten += Deno.stderr.writeSync(data.subarray(bytesWritten));
+  }
   Deno.exit(0);
 });

tools/wpt/utils.ts

@@ -6,6 +6,7 @@ import { join, ROOT_PATH } from "../util.js";
 
 export const {
   json,
+  wptreport,
   quiet,
   release,
   rebuild,
@@ -14,7 +15,7 @@ export const {
 } = parse(Deno.args, {
   "--": true,
   boolean: ["quiet", "release", "no-interactive"],
-  string: ["json"],
+  string: ["json", "wptreport"],
 });
 
 /// PAGE ROOT
@@ -145,3 +146,54 @@ export async function cargoBuild() {
   proc.close();
   assert(status.success, "cargo build failed");
 }
+
+/// WPTREPORT
+
+export async function generateRunInfo(): Promise<unknown> {
+  const oses = {
+    "windows": "win",
+    "darwin": "mac",
+    "linux": "linux",
+  };
+  const proc = Deno.run({
+    cmd: ["git", "rev-parse", "HEAD"],
+    cwd: join(ROOT_PATH, "test_util", "wpt"),
+    stdout: "piped",
+  });
+  await proc.status();
+  const revision = (new TextDecoder().decode(await proc.output())).trim();
+  proc.close();
+  const proc2 = Deno.run({
+    cmd: [
+      join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`),
+      "eval",
+      "console.log(JSON.stringify(Deno.version))",
+    ],
+    cwd: join(ROOT_PATH, "test_util", "wpt"),
+    stdout: "piped",
+  });
+  await proc2.status();
+  const version = JSON.parse(new TextDecoder().decode(await proc2.output()));
+  proc2.close();
+  const runInfo = {
+    "os": oses[Deno.build.os],
+    "processor": Deno.build.arch,
+    "version": "unknown",
+    "os_version": "unknown",
+    "bits": 64,
+    "has_sandbox": true,
+    "webrender": false,
+    "automation": false,
+    "linux_distro": "unknown",
+    "revision": revision,
+    "python_version": 3,
+    "product": "deno",
+    "debug": false,
+    "browser_version": version.deno,
+    "browser_channel": version.deno.includes("+") ? "canary" : "stable",
+    "verify": false,
+    "wasm": false,
+    "headless": true,
+  };
+  return runInfo;
+}