#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // This script is used to run WPT tests for Deno. import { runSingleTest, runWithTestUtil, TestCaseResult, TestResult, } from "./wpt/runner.ts"; import { assert, autoConfig, cargoBuild, checkPy3Available, escapeLoneSurrogates, Expectation, generateRunInfo, getExpectation, getExpectFailForCase, getManifest, inspectBrk, json, ManifestFolder, ManifestTestOptions, ManifestTestVariation, noIgnore, quiet, rest, runPy, updateManifest, wptreport, } from "./wpt/utils.ts"; import { pooledMap } from "../test_util/std/async/pool.ts"; import { blue, bold, green, red, yellow } from "../test_util/std/fmt/colors.ts"; import { writeAll, writeAllSync } from "../test_util/std/streams/write_all.ts"; import { saveExpectation } from "./wpt/utils.ts"; class TestFilter { filter?: string[]; constructor(filter?: string[]) { this.filter = filter; } matches(path: string): boolean { if (this.filter === undefined || this.filter.length == 0) { return true; } for (const filter of this.filter) { if (filter.startsWith("/")) { if (path.startsWith(filter)) { return true; } } else { if (path.substring(1).startsWith(filter)) { return true; } } } return false; } } const command = Deno.args[0]; switch (command) { case "setup": await checkPy3Available(); await updateManifest(); await setup(); break; case "run": await cargoBuild(); await run(); break; case "update": await cargoBuild(); await update(); break; default: console.log(`Possible commands: setup Validate that your environment is configured correctly, or help you configure it. run Run all tests like specified in \`expectation.json\`. update Update the \`expectation.json\` to match the current reality. More details at https://deno.land/manual@main/contributing/web_platform_tests `); break; } async function setup() { const hostsPath = Deno.build.os == "windows" ? `${Deno.env.get("SystemRoot")}\\System32\\drivers\\etc\\hosts` : "/etc/hosts"; // TODO(lucacsonato): use this when 1.7.1 is released. // const records = await Deno.resolveDns("web-platform.test", "A"); // const etcHostsConfigured = records[0] == "127.0.0.1"; const hostsFile = await Deno.readTextFile(hostsPath); const etcHostsConfigured = hostsFile.includes("web-platform.test"); if (etcHostsConfigured) { console.log(hostsPath + " is already configured."); } else { const autoConfigure = autoConfig || confirm( `The WPT require certain entries to be present in your ${hostsPath} file. Should these be configured automatically?`, ); if (autoConfigure) { const { success, stdout } = await runPy(["wpt", "make-hosts-file"], { stdout: "piped", }).output(); assert(success, "wpt make-hosts-file should not fail"); const entries = new TextDecoder().decode(stdout); const file = await Deno.open(hostsPath, { append: true }).catch((err) => { if (err instanceof Deno.errors.PermissionDenied) { throw new Error( `Failed to open ${hostsPath} (permission error). Please run this command again with sudo, or configure the entries manually.`, ); } else { throw err; } }); await writeAll( file, new TextEncoder().encode( "\n\n# Configured for Web Platform Tests (Deno)\n" + entries, ), ); console.log(`Updated ${hostsPath}`); } else { console.log(`Please configure the ${hostsPath} entries manually.`); if (Deno.build.os == "windows") { console.log("To do this run the following command in PowerShell:"); console.log(""); console.log(" cd test_util/wpt/"); console.log( " python.exe wpt make-hosts-file | Out-File $env:SystemRoot\\System32\\drivers\\etc\\hosts -Encoding ascii -Append", ); console.log(""); } else { console.log("To do this run the following command in your shell:"); console.log(""); console.log(" cd test_util/wpt/"); console.log( " python3 ./wpt make-hosts-file | sudo tee -a /etc/hosts", ); console.log(""); } } } console.log(green("Setup complete!")); } interface TestToRun { path: string; url: URL; options: ManifestTestOptions; expectation: boolean | string[]; } function getTestTimeout(test: TestToRun) { if (Deno.env.get("CI")) { // Don't give expected failures the full time if (test.expectation === false) { return { long: 60_000, default: 10_000 }; } return { long: 4 * 60_000, default: 4 * 60_000 }; } return { long: 60_000, default: 10_000 }; } async function run() { const startTime = new Date().getTime(); assert(Array.isArray(rest), "filter must be array"); const expectation = getExpectation(); const filter = new TestFilter(rest); const tests = discoverTestsToRun( filter, expectation, ); assertAllExpectationsHaveTests(expectation, tests, filter); const cores = navigator.hardwareConcurrency; console.log(`Going to run ${tests.length} test files on ${cores} cores.`); const results = await runWithTestUtil(false, async () => { const results: { test: TestToRun; result: TestResult }[] = []; const inParallel = !(cores === 1 || tests.length === 1); // ideally we would parallelize all tests, but we ran into some flakiness // on the CI, so here we're partitioning based on the start of the test path const partitionedTests = partitionTests(tests); const iter = pooledMap(cores, partitionedTests, async (tests) => { for (const test of tests) { if (!inParallel) { console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`); } const result = await runSingleTest( test.url, test.options, inParallel ? () => {} : createReportTestCase(test.expectation), inspectBrk, getTestTimeout(test), ); results.push({ test, result }); if (inParallel) { console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`); } reportVariation(result, test.expectation); } }); for await (const _ of iter) { // ignore } return results; }); const endTime = new Date().getTime(); if (json) { const minifiedResults = []; for (const result of results) { const minified = { file: result.test.path, name: Object.fromEntries(result.test.options.script_metadata ?? []).title ?? null, cases: result.result.cases.map((case_) => ({ name: case_.name, passed: case_.passed, })), }; minifiedResults.push(minified); } await Deno.writeTextFile(json, JSON.stringify(minifiedResults)); } if (wptreport) { const report = await generateWptReport(results, startTime, endTime); await Deno.writeTextFile(wptreport, JSON.stringify(report)); } const code = reportFinal(results, endTime - startTime); Deno.exit(code); } async function generateWptReport( results: { test: TestToRun; result: TestResult }[], startTime: number, endTime: number, ) { const runInfo = await generateRunInfo(); const reportResults = []; for (const { test, result } of results) { const status = result.status !== 0 ? "CRASH" : result.harnessStatus?.status === 0 ? "OK" : "ERROR"; let message; if (result.harnessStatus === null && result.status === 0) { // If the only error is the event loop running out of tasks, using stderr // as the message won't help. message = "Event loop run out of tasks."; } else { message = result.harnessStatus?.message ?? (result.stderr.trim() || null); } const reportResult = { test: test.url.pathname + test.url.search + test.url.hash, subtests: result.cases.map((case_) => { let expected = undefined; if (!case_.passed) { if (typeof test.expectation === "boolean") { expected = test.expectation ? "PASS" : "FAIL"; } else if (Array.isArray(test.expectation)) { expected = test.expectation.includes(case_.name) ? "FAIL" : "PASS"; } else { expected = "PASS"; } } return { name: escapeLoneSurrogates(case_.name), status: case_.passed ? "PASS" : "FAIL", message: escapeLoneSurrogates(case_.message), expected, known_intermittent: [], }; }), status, message: escapeLoneSurrogates(message), duration: result.duration, expected: status === "OK" ? undefined : "OK", "known_intermittent": [], }; reportResults.push(reportResult); } return { "run_info": runInfo, "time_start": startTime, "time_end": endTime, "results": reportResults, }; } // Check that all expectations in the expectations file have a test that will be // run. function assertAllExpectationsHaveTests( expectation: Expectation, testsToRun: TestToRun[], filter: TestFilter, ): void { const tests = new Set(testsToRun.map((t) => t.path)); const missingTests: string[] = []; function walk(parentExpectation: Expectation, parent: string) { for (const [key, expectation] of Object.entries(parentExpectation)) { const path = `${parent}/${key}`; if (!filter.matches(path)) continue; if ( (typeof expectation == "boolean" || Array.isArray(expectation)) && key !== "ignore" ) { if (!tests.has(path)) { missingTests.push(path); } } else { walk(expectation, path); } } } walk(expectation, ""); if (missingTests.length > 0) { console.log( red( "Following tests are missing in manifest, but are present in expectations:", ), ); console.log(""); console.log(missingTests.join("\n")); Deno.exit(1); } } async function update() { assert(Array.isArray(rest), "filter must be array"); const startTime = new Date().getTime(); const filter = new TestFilter(rest); const tests = discoverTestsToRun(filter, true); console.log(`Going to run ${tests.length} test files.`); const results = await runWithTestUtil(false, async () => { const results = []; for (const test of tests) { console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`); const result = await runSingleTest( test.url, test.options, json ? () => {} : createReportTestCase(test.expectation), inspectBrk, { long: 60_000, default: 10_000 }, ); results.push({ test, result }); reportVariation(result, test.expectation); } return results; }); const endTime = new Date().getTime(); if (json) { await Deno.writeTextFile(json, JSON.stringify(results)); } const resultTests: Record< string, { passed: string[]; failed: string[]; testSucceeded: boolean } > = {}; for (const { test, result } of results) { if (!resultTests[test.path]) { resultTests[test.path] = { passed: [], failed: [], testSucceeded: result.status === 0 && result.harnessStatus !== null, }; } for (const case_ of result.cases) { if (case_.passed) { resultTests[test.path].passed.push(case_.name); } else { resultTests[test.path].failed.push(case_.name); } } } const currentExpectation = getExpectation(); for (const [path, result] of Object.entries(resultTests)) { const { passed, failed, testSucceeded } = result; let finalExpectation: boolean | string[]; if (failed.length == 0 && testSucceeded) { finalExpectation = true; } else if (failed.length > 0 && passed.length > 0 && testSucceeded) { finalExpectation = failed; } else { finalExpectation = false; } insertExpectation( path.slice(1).split("/"), currentExpectation, finalExpectation, ); } saveExpectation(currentExpectation); reportFinal(results, endTime - startTime); console.log(blue("Updated expectation.json to match reality.")); Deno.exit(0); } function insertExpectation( segments: string[], currentExpectation: Expectation, finalExpectation: boolean | string[], ) { const segment = segments.shift(); assert(segment, "segments array must never be empty"); if (segments.length > 0) { if ( !currentExpectation[segment] || Array.isArray(currentExpectation[segment]) || typeof currentExpectation[segment] === "boolean" ) { currentExpectation[segment] = {}; } insertExpectation( segments, currentExpectation[segment] as Expectation, finalExpectation, ); } else { currentExpectation[segment] = finalExpectation; } } function reportFinal( results: { test: TestToRun; result: TestResult }[], duration: number, ): number { const finalTotalCount = results.length; let finalFailedCount = 0; const finalFailed: [string, TestCaseResult][] = []; let finalExpectedFailedAndFailedCount = 0; const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = []; const finalExpectedFailedButPassedFiles: string[] = []; const finalFailedFiles: string[] = []; for (const { test, result } of results) { const { failed, failedCount, expectedFailedButPassed, expectedFailedAndFailedCount, } = analyzeTestResult( result, test.expectation, ); if (result.status !== 0 || result.harnessStatus === null) { if (test.expectation === false) { finalExpectedFailedAndFailedCount += 1; } else { finalFailedCount += 1; finalFailedFiles.push(test.path); } } else if (failedCount > 0) { finalFailedCount += 1; for (const case_ of failed) { finalFailed.push([test.path, case_]); } for (const case_ of expectedFailedButPassed) { finalExpectedFailedButPassedTests.push([test.path, case_]); } } else if ( test.expectation === false && expectedFailedAndFailedCount != result.cases.length ) { finalExpectedFailedButPassedFiles.push(test.path); } } const finalPassedCount = finalTotalCount - finalFailedCount; console.log(bold(blue("=".repeat(40)))); if (finalFailed.length > 0) { console.log(`\nfailures:\n`); } for (const result of finalFailed) { console.log( ` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`, ); } if (finalFailedFiles.length > 0) { console.log(`\nfile failures:\n`); } for (const result of finalFailedFiles) { console.log( ` ${JSON.stringify(result)}`, ); } if (finalExpectedFailedButPassedTests.length > 0) { console.log(`\nexpected test failures that passed:\n`); } for (const result of finalExpectedFailedButPassedTests) { console.log( ` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`, ); } if (finalExpectedFailedButPassedFiles.length > 0) { console.log(`\nexpected file failures that passed:\n`); } for (const result of finalExpectedFailedButPassedFiles) { console.log(` ${JSON.stringify(result)}`); } const failed = (finalFailedCount > 0) || (finalExpectedFailedButPassedFiles.length > 0); console.log( `\nfinal result: ${ failed ? red("failed") : green("ok") }. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount} (${duration}ms)\n`, ); return failed ? 1 : 0; } function analyzeTestResult( result: TestResult, expectation: boolean | string[], ): { failed: TestCaseResult[]; failedCount: number; passedCount: number; totalCount: number; expectedFailedButPassed: TestCaseResult[]; expectedFailedButPassedCount: number; expectedFailedAndFailedCount: number; } { const failed = result.cases.filter( (t) => !getExpectFailForCase(expectation, t.name) && !t.passed, ); const expectedFailedButPassed = result.cases.filter( (t) => getExpectFailForCase(expectation, t.name) && t.passed, ); const expectedFailedButPassedCount = expectedFailedButPassed.length; const failedCount = failed.length + expectedFailedButPassedCount; const expectedFailedAndFailedCount = result.cases.filter( (t) => getExpectFailForCase(expectation, t.name) && !t.passed, ).length; const totalCount = result.cases.length; const passedCount = totalCount - failedCount - expectedFailedAndFailedCount; return { failed, failedCount, passedCount, totalCount, expectedFailedButPassed, expectedFailedButPassedCount, expectedFailedAndFailedCount, }; } function reportVariation(result: TestResult, expectation: boolean | string[]) { if (result.status !== 0 || result.harnessStatus === null) { if (result.stderr) { console.log(`test stderr:\n${result.stderr}\n`); } const expectFail = expectation === false; const failReason = result.status !== 0 ? "runner failed during test" : "the event loop run out of tasks during the test"; console.log( `\nfile result: ${ expectFail ? yellow("failed (expected)") : red("failed") }. ${failReason} (${formatDuration(result.duration)})\n`, ); return; } const { failed, failedCount, passedCount, totalCount, expectedFailedButPassed, expectedFailedButPassedCount, expectedFailedAndFailedCount, } = analyzeTestResult(result, expectation); if (failed.length > 0) { console.log(`\nfailures:`); } for (const result of failed) { console.log(`\n${result.name}\n${result.message}\n${result.stack}`); } if (failedCount > 0) { console.log(`\nfailures:\n`); } for (const result of failed) { console.log(` ${JSON.stringify(result.name)}`); } if (expectedFailedButPassedCount > 0) { console.log(`\nexpected failures that passed:\n`); } for (const result of expectedFailedButPassed) { console.log(` ${JSON.stringify(result.name)}`); } if (result.stderr) { console.log("\ntest stderr:\n" + result.stderr); } console.log( `\nfile result: ${ failedCount > 0 ? red("failed") : green("ok") }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount} (${ formatDuration(result.duration) })\n`, ); } function createReportTestCase(expectation: boolean | string[]) { return function reportTestCase({ name, status }: TestCaseResult) { const expectFail = getExpectFailForCase(expectation, name); let simpleMessage = `test ${name} ... `; switch (status) { case 0: if (expectFail) { simpleMessage += red("ok (expected fail)"); } else { simpleMessage += green("ok"); if (quiet) { // don't print `ok` tests if --quiet is enabled return; } } break; case 1: if (expectFail) { simpleMessage += yellow("failed (expected)"); } else { simpleMessage += red("failed"); } break; case 2: if (expectFail) { simpleMessage += yellow("failed (expected)"); } else { simpleMessage += red("failed (timeout)"); } break; case 3: if (expectFail) { simpleMessage += yellow("failed (expected)"); } else { simpleMessage += red("failed (incomplete)"); } break; } writeAllSync(Deno.stdout, new TextEncoder().encode(simpleMessage + "\n")); }; } function discoverTestsToRun( filter: TestFilter, expectation: Expectation | string[] | boolean = getExpectation(), ): TestToRun[] { const manifestFolder = getManifest().items.testharness; const testsToRun: TestToRun[] = []; function walk( parentFolder: ManifestFolder, parentExpectation: Expectation | string[] | boolean, prefix: string, ) { for (const [key, entry] of Object.entries(parentFolder)) { if (Array.isArray(entry)) { for ( const [path, options] of entry.slice( 1, ) as ManifestTestVariation[] ) { // Test keys ending with ".html" include their own html boilerplate. // Test keys ending with ".js" will have the necessary boilerplate generated and // the manifest path will contain the full path to the generated html test file. // See: https://web-platform-tests.org/writing-tests/testharness.html if (!key.endsWith(".html") && !key.endsWith(".js")) continue; const testHtmlPath = path ?? `${prefix}/${key}`; const url = new URL(testHtmlPath, "http://web-platform.test:8000"); if (!url.pathname.endsWith(".html")) { continue; } // These tests require an HTTP2 compatible server. if (url.pathname.includes(".h2.")) { continue; } // Streaming fetch requests need a server that supports chunked // encoding, which the WPT test server does not. Unfortunately this // also disables some useful fetch tests. if (url.pathname.includes("request-upload")) { continue; } const finalPath = url.pathname + url.search; const split = finalPath.split("/"); const finalKey = split[split.length - 1]; const expectation = Array.isArray(parentExpectation) || typeof parentExpectation == "boolean" ? parentExpectation : parentExpectation[finalKey]; if (expectation === undefined) continue; if (typeof expectation === "object") { if (typeof expectation.ignore !== "undefined") { assert( typeof expectation.ignore === "boolean", "test entry's `ignore` key must be a boolean", ); if (expectation.ignore === true && !noIgnore) continue; } } if (!noIgnore) { assert( Array.isArray(expectation) || typeof expectation == "boolean", "test entry must not have a folder expectation", ); } if (!filter.matches(finalPath)) continue; testsToRun.push({ path: finalPath, url, options, expectation, }); } } else { const expectation = Array.isArray(parentExpectation) || typeof parentExpectation == "boolean" ? parentExpectation : parentExpectation[key]; if (expectation === undefined) continue; walk(entry, expectation, `${prefix}/${key}`); } } } walk(manifestFolder, expectation, ""); return testsToRun; } function partitionTests(tests: TestToRun[]): TestToRun[][] { const testsByKey: { [key: string]: TestToRun[] } = {}; for (const test of tests) { // Run all WebCryptoAPI tests in parallel if (test.path.includes("/WebCryptoAPI")) { testsByKey[test.path] = [test]; continue; } // Paths looks like: /fetch/corb/img-html-correctly-labeled.sub-ref.html const key = test.path.split("/")[1]; if (!(key in testsByKey)) { testsByKey[key] = []; } testsByKey[key].push(test); } return Object.values(testsByKey); } function formatDuration(duration: number): string { if (duration >= 5000) { return red(`${duration}ms`); } else if (duration >= 1000) { return yellow(`${duration}ms`); } else { return `${duration}ms`; } }