From f1deed41e7cc04440a5fb8cdae486ae00513a361 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Sun, 6 Jun 2021 18:08:50 +0200 Subject: [PATCH] tests: generate and upload wptreport.json (#10869) These reports can be consumed by tools like `wptreport` or https://wpt.fyi. The old style report could be removed in a future PR when wpt.deno.land is updated. --- .github/workflows/ci.yml | 7 ++-- tools/wpt.ts | 60 ++++++++++++++++++++++++++++++++++ tools/wpt/runner.ts | 18 ++++++++++ tools/wpt/testharnessreport.js | 9 ++++- tools/wpt/utils.ts | 54 +++++++++++++++++++++++++++++- 5 files changed, 142 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91cbf53ae9..b86d54d68c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -177,8 +177,7 @@ jobs: ~/.cargo/registry/index ~/.cargo/registry/cache ~/.cargo/git/db - key: - a-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} + key: a-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} # In main branch, always creates fresh cache - name: Cache build output (main) @@ -308,7 +307,7 @@ jobs: if: startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' && matrix.profile == 'release' run: | deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts setup - deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts run --quiet --release --json=wpt.json + deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run ./tools/wpt.ts run --quiet --release --json=wpt.json --wptreport=wptreport.json - name: Upload wpt results to dl.deno.land if: | @@ -319,6 +318,7 @@ jobs: github.ref == 'refs/heads/main' run: | gsutil cp ./wpt.json gs://dl.deno.land/wpt/$(git rev-parse HEAD).json + gsutil cp ./wptreport.json gs://dl.deno.land/wpt/$(git rev-parse HEAD)-wptreport.json echo $(git rev-parse HEAD) > wpt-latest.txt gsutil cp wpt-latest.txt gs://dl.deno.land/wpt-latest.txt @@ -407,4 +407,3 @@ jobs: rm -rf target/*/examples/ rm -rf target/*/gn_out/ rm -rf target/*/*.zip - diff --git a/tools/wpt.ts b/tools/wpt.ts index e6f08dd34b..8f82488806 100755 --- a/tools/wpt.ts +++ b/tools/wpt.ts @@ -15,6 +15,7 @@ import { cargoBuild, checkPy3Available, Expectation, + generateRunInfo, getExpectation, getExpectFailForCase, getManifest, @@ -26,6 +27,7 @@ import { rest, runPy, updateManifest, + wptreport, } from "./wpt/utils.ts"; import { blue, @@ -148,6 +150,7 @@ interface TestToRun { } async function run() { + const startTime = new Date().getTime(); assert(Array.isArray(rest), "filter must be array"); const expectation = getExpectation(); const tests = discoverTestsToRun( @@ -173,6 +176,7 @@ async function run() { return results; }); + const endTime = new Date().getTime(); if (json) { const minifiedResults = []; @@ -191,10 +195,66 @@ async function run() { } await Deno.writeTextFile(json, JSON.stringify(minifiedResults)); } + + if (wptreport) { + const report = await generateWptreport(results, startTime, endTime); + await Deno.writeTextFile(wptreport, JSON.stringify(report)); + } + const code = reportFinal(results); Deno.exit(code); } +async function generateWptreport( + results: { test: TestToRun; result: TestResult }[], + startTime: number, + endTime: number, +) { + const runInfo = await generateRunInfo(); + const reportResults = []; + for (const { test, result } of results) { + const status = result.status !== 0 + ? "CRASH" + : result.harnessStatus?.status === 0 + ? "OK" + : "ERROR"; + const reportResult = { + test: test.url.pathname + test.url.search + test.url.hash, + subtests: result.cases.map((case_) => { + let expected = undefined; + if (!case_.passed) { + if (typeof test.expectation === "boolean") { + expected = test.expectation ? "PASS" : "FAIL"; + } else { + expected = test.expectation.includes(case_.name) ? "FAIL" : "PASS"; + } + } + + return { + name: case_.name, + status: case_.passed ? "PASS" : "FAIL", + message: case_.message, + expected, + known_intermittent: [], + }; + }), + status, + message: result.harnessStatus?.message ?? + (result.stderr.trim() || null), + duration: result.duration, + expected: status === "OK" ? undefined : "OK", + "known_intermittent": [], + }; + reportResults.push(reportResult); + } + return { + "run_info": runInfo, + "time_start": startTime, + "time_end": endTime, + "results": reportResults, + }; +} + // Check that all expectations in the expectations file have a test that will be // run. function assertAllExpectationsHaveTests( diff --git a/tools/wpt/runner.ts b/tools/wpt/runner.ts index a0941b5210..dcc88a1231 100644 --- a/tools/wpt/runner.ts +++ b/tools/wpt/runner.ts @@ -47,10 +47,18 @@ export async function runWithTestUtil( export interface TestResult { cases: TestCaseResult[]; + harnessStatus: TestHarnessStatus | null; + duration: number; status: number; stderr: string; } +export interface TestHarnessStatus { + status: number; + message: string | null; + stack: string | null; +} + export interface TestCaseResult { name: string; passed: boolean; @@ -71,6 +79,8 @@ export async function runSingleTest( }); await Deno.writeTextFile(tempFile, bundle); + const startTime = new Date().getTime(); + const proc = Deno.run({ cmd: [ join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`), @@ -94,6 +104,8 @@ export async function runSingleTest( const cases = []; let stderr = ""; + let harnessStatus = null; + const lines = readLines(proc.stderr); for await (const line of lines) { if (line.startsWith("{")) { @@ -101,15 +113,21 @@ export async function runSingleTest( const result = { ...data, passed: data.status == 0 }; cases.push(result); reporter(result); + } else if (line.startsWith("#$#$#{")) { + harnessStatus = JSON.parse(line.slice(5)); } else { stderr += line + "\n"; console.error(stderr); } } + const duration = new Date().getTime() - startTime; + const { code } = await proc.status(); return { status: code, + harnessStatus, + duration, cases, stderr, }; diff --git a/tools/wpt/testharnessreport.js b/tools/wpt/testharnessreport.js index d3e7833767..04251c8524 100644 --- a/tools/wpt/testharnessreport.js +++ b/tools/wpt/testharnessreport.js @@ -10,6 +10,13 @@ window.add_result_callback(({ message, name, stack, status }) => { } }); -window.add_completion_callback((_tests, _harnessStatus) => { +window.add_completion_callback((_tests, harnessStatus) => { + const data = new TextEncoder().encode( + `#$#$#${JSON.stringify(harnessStatus)}\n`, + ); + let bytesWritten = 0; + while (bytesWritten < data.byteLength) { + bytesWritten += Deno.stderr.writeSync(data.subarray(bytesWritten)); + } Deno.exit(0); }); diff --git a/tools/wpt/utils.ts b/tools/wpt/utils.ts index cb454300b7..3b1eb9965f 100644 --- a/tools/wpt/utils.ts +++ b/tools/wpt/utils.ts @@ -6,6 +6,7 @@ import { join, ROOT_PATH } from "../util.js"; export const { json, + wptreport, quiet, release, rebuild, @@ -14,7 +15,7 @@ export const { } = parse(Deno.args, { "--": true, boolean: ["quiet", "release", "no-interactive"], - string: ["json"], + string: ["json", "wptreport"], }); /// PAGE ROOT @@ -145,3 +146,54 @@ export async function cargoBuild() { proc.close(); assert(status.success, "cargo build failed"); } + +/// WPTREPORT + +export async function generateRunInfo(): Promise { + const oses = { + "windows": "win", + "darwin": "mac", + "linux": "linux", + }; + const proc = Deno.run({ + cmd: ["git", "rev-parse", "HEAD"], + cwd: join(ROOT_PATH, "test_util", "wpt"), + stdout: "piped", + }); + await proc.status(); + const revision = (new TextDecoder().decode(await proc.output())).trim(); + proc.close(); + const proc2 = Deno.run({ + cmd: [ + join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`), + "eval", + "console.log(JSON.stringify(Deno.version))", + ], + cwd: join(ROOT_PATH, "test_util", "wpt"), + stdout: "piped", + }); + await proc2.status(); + const version = JSON.parse(new TextDecoder().decode(await proc2.output())); + proc2.close(); + const runInfo = { + "os": oses[Deno.build.os], + "processor": Deno.build.arch, + "version": "unknown", + "os_version": "unknown", + "bits": 64, + "has_sandbox": true, + "webrender": false, + "automation": false, + "linux_distro": "unknown", + "revision": revision, + "python_version": 3, + "product": "deno", + "debug": false, + "browser_version": version.deno, + "browser_channel": version.deno.includes("+") ? "canary" : "stable", + "verify": false, + "wasm": false, + "headless": true, + }; + return runInfo; +}