1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00
denoland-deno/tools/wpt.ts
crowlKats dfe528198d
feat: add WebStorage API (#7819)
This commit introduces localStorage and sessionStorage.
2021-05-10 12:02:47 +02:00

601 lines
16 KiB
TypeScript
Executable file

#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
// This script is used to run WPT tests for Deno.
import {
runSingleTest,
runWithTestUtil,
TestCaseResult,
TestResult,
} from "./wpt/runner.ts";
import {
assert,
autoConfig,
cargoBuild,
checkPy3Available,
Expectation,
getExpectation,
getExpectFailForCase,
getManifest,
json,
ManifestFolder,
ManifestTestOptions,
ManifestTestVariation,
quiet,
rest,
runPy,
updateManifest,
} from "./wpt/utils.ts";
import {
blue,
bold,
green,
red,
yellow,
} from "https://deno.land/std@0.84.0/fmt/colors.ts";
import {
writeAll,
writeAllSync,
} from "https://deno.land/std@0.95.0/io/util.ts";
import { saveExpectation } from "./wpt/utils.ts";
const command = Deno.args[0];
switch (command) {
case "setup":
await checkPy3Available();
await updateManifest();
await setup();
break;
case "run":
await cargoBuild();
await run();
break;
case "update":
await cargoBuild();
await update();
break;
default:
console.log(`Possible commands:
setup
Validate that your environment is configured correctly, or help you configure it.
run
Run all tests like specified in \`expectation.json\`.
update
Update the \`expectation.json\` to match the current reality.
More details at https://deno.land/manual@main/contributing/web_platform_tests
`);
break;
}
async function setup() {
// TODO(lucacsonato): use this when 1.7.1 is released.
// const records = await Deno.resolveDns("web-platform.test", "A");
// const etcHostsConfigured = records[0] == "127.0.0.1";
const hostsFile = await Deno.readTextFile("/etc/hosts");
const etcHostsConfigured = hostsFile.includes("web-platform.test");
if (etcHostsConfigured) {
console.log("/etc/hosts is already configured.");
} else {
const autoConfigure = autoConfig ||
confirm(
"The WPT require certain entries to be present in your /etc/hosts file. Should these be configured automatically?",
);
if (autoConfigure) {
const proc = runPy(["wpt", "make-hosts-file"], { stdout: "piped" });
const status = await proc.status();
assert(status.success, "wpt make-hosts-file should not fail");
const entries = new TextDecoder().decode(await proc.output());
const hostsPath = Deno.build.os == "windows"
? `${Deno.env.get("SystemRoot")}\\System32\\drivers\\etc\\hosts`
: "/etc/hosts";
const file = await Deno.open(hostsPath, { append: true }).catch((err) => {
if (err instanceof Deno.errors.PermissionDenied) {
throw new Error(
`Failed to open ${hostsPath} (permission error). Please run this command again with sudo, or configure the entries manually.`,
);
} else {
throw err;
}
});
await writeAll(
file,
new TextEncoder().encode(
"\n\n# Configured for Web Platform Tests (Deno)\n" + entries,
),
);
console.log("Updated /etc/hosts");
} else {
console.log("Please configure the /etc/hosts entries manually.");
if (Deno.build.os == "windows") {
console.log("To do this run the following command in PowerShell:");
console.log("");
console.log(" cd test_util/wpt/");
console.log(
" python.exe wpt make-hosts-file | Out-File $env:SystemRoot\\System32\\drivers\\etc\\hosts -Encoding ascii -Append",
);
console.log("");
} else {
console.log("To do this run the following command in your shell:");
console.log("");
console.log(" cd test_util/wpt/");
console.log(
" python3 ./wpt make-hosts-file | sudo tee -a /etc/hosts",
);
console.log("");
}
}
}
console.log(green("Setup complete!"));
}
interface TestToRun {
path: string;
url: URL;
options: ManifestTestOptions;
expectation: boolean | string[];
}
async function run() {
assert(Array.isArray(rest), "filter must be array");
const expectation = getExpectation();
const tests = discoverTestsToRun(
rest.length == 0 ? undefined : rest,
expectation,
);
assertAllExpectationsHaveTests(expectation, tests, rest);
console.log(`Going to run ${tests.length} test files.`);
const results = await runWithTestUtil(false, async () => {
const results = [];
for (const test of tests) {
console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
const result = await runSingleTest(
test.url,
test.options,
json ? () => {} : createReportTestCase(test.expectation),
);
results.push({ test, result });
reportVariation(result, test.expectation);
}
return results;
});
if (json) {
await Deno.writeTextFile(json, JSON.stringify(results));
}
const code = reportFinal(results);
Deno.exit(code);
}
// Check that all expectations in the expectations file have a test that will be
// run.
function assertAllExpectationsHaveTests(
expectation: Expectation,
testsToRun: TestToRun[],
filter?: string[],
): void {
const tests = new Set(testsToRun.map((t) => t.path));
const missingTests: string[] = [];
function walk(parentExpectation: Expectation, parent: string) {
for (const key in parentExpectation) {
const path = `${parent}/${key}`;
if (
filter &&
!filter.find((filter) => path.substring(1).startsWith(filter))
) {
continue;
}
const expectation = parentExpectation[key];
if (typeof expectation == "boolean" || Array.isArray(expectation)) {
if (!tests.has(path)) {
missingTests.push(path);
}
} else {
walk(expectation, path);
}
}
}
walk(expectation, "");
if (missingTests.length > 0) {
console.log(
red(
"Following tests are missing in manifest, but are present in expectations:",
),
);
console.log("");
console.log(missingTests.join("\n"));
Deno.exit(1);
}
}
async function update() {
assert(Array.isArray(rest), "filter must be array");
const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
console.log(`Going to run ${tests.length} test files.`);
const results = await runWithTestUtil(false, async () => {
const results = [];
for (const test of tests) {
console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
const result = await runSingleTest(
test.url,
test.options,
json ? () => {} : createReportTestCase(test.expectation),
);
results.push({ test, result });
reportVariation(result, test.expectation);
}
return results;
});
if (json) {
await Deno.writeTextFile(json, JSON.stringify(results));
}
const resultTests: Record<
string,
{ passed: string[]; failed: string[]; status: number }
> = {};
for (const { test, result } of results) {
if (!resultTests[test.path]) {
resultTests[test.path] = {
passed: [],
failed: [],
status: result.status,
};
}
for (const case_ of result.cases) {
if (case_.passed) {
resultTests[test.path].passed.push(case_.name);
} else {
resultTests[test.path].failed.push(case_.name);
}
}
}
const currentExpectation = getExpectation();
for (const path in resultTests) {
const { passed, failed, status } = resultTests[path];
let finalExpectation: boolean | string[];
if (failed.length == 0 && status == 0) {
finalExpectation = true;
} else if (failed.length > 0 && passed.length > 0 && status == 0) {
finalExpectation = failed;
} else {
finalExpectation = false;
}
insertExpectation(
path.slice(1).split("/"),
currentExpectation,
finalExpectation,
);
}
saveExpectation(currentExpectation);
reportFinal(results);
console.log(blue("Updated expectation.json to match reality."));
Deno.exit(0);
}
function insertExpectation(
segments: string[],
currentExpectation: Expectation,
finalExpectation: boolean | string[],
) {
const segment = segments.shift();
assert(segment, "segments array must never be empty");
if (segments.length > 0) {
if (
!currentExpectation[segment] ||
Array.isArray(currentExpectation[segment]) ||
typeof currentExpectation[segment] === "boolean"
) {
currentExpectation[segment] = {};
}
insertExpectation(
segments,
currentExpectation[segment] as Expectation,
finalExpectation,
);
} else {
currentExpectation[segment] = finalExpectation;
}
}
function reportFinal(
results: { test: TestToRun; result: TestResult }[],
): number {
const finalTotalCount = results.length;
let finalFailedCount = 0;
const finalFailed: [string, TestCaseResult][] = [];
let finalExpectedFailedAndFailedCount = 0;
const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = [];
const finalExpectedFailedButPassedFiles: string[] = [];
for (const { test, result } of results) {
const { failed, failedCount, expectedFailedButPassed } = analyzeTestResult(
result,
test.expectation,
);
if (result.status !== 0) {
if (test.expectation === false) {
finalExpectedFailedAndFailedCount += 1;
} else {
finalFailedCount += 1;
finalExpectedFailedButPassedFiles.push(test.path);
}
} else if (failedCount > 0) {
finalFailedCount += 1;
for (const case_ of failed) {
finalFailed.push([test.path, case_]);
}
for (const case_ of expectedFailedButPassed) {
finalExpectedFailedButPassedTests.push([test.path, case_]);
}
}
}
const finalPassedCount = finalTotalCount - finalFailedCount;
console.log(bold(blue("=".repeat(40))));
if (finalFailed.length > 0) {
console.log(`\nfailures:\n`);
}
for (const result of finalFailed) {
console.log(
` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
);
}
if (finalExpectedFailedButPassedTests.length > 0) {
console.log(`\nexpected test failures that passed:\n`);
}
for (const result of finalExpectedFailedButPassedTests) {
console.log(
` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
);
}
if (finalExpectedFailedButPassedFiles.length > 0) {
console.log(`\nexpected file failures that passed:\n`);
}
for (const result of finalExpectedFailedButPassedFiles) {
console.log(` ${JSON.stringify(result)}`);
}
console.log(
`\nfinal result: ${
finalFailedCount > 0 ? red("failed") : green("ok")
}. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount}\n`,
);
return finalFailedCount > 0 ? 1 : 0;
}
function analyzeTestResult(
result: TestResult,
expectation: boolean | string[],
): {
failed: TestCaseResult[];
failedCount: number;
passedCount: number;
totalCount: number;
expectedFailedButPassed: TestCaseResult[];
expectedFailedButPassedCount: number;
expectedFailedAndFailedCount: number;
} {
const failed = result.cases.filter(
(t) => !getExpectFailForCase(expectation, t.name) && !t.passed,
);
const expectedFailedButPassed = result.cases.filter(
(t) => getExpectFailForCase(expectation, t.name) && t.passed,
);
const expectedFailedButPassedCount = expectedFailedButPassed.length;
const failedCount = failed.length + expectedFailedButPassedCount;
const expectedFailedAndFailedCount = result.cases.filter(
(t) => getExpectFailForCase(expectation, t.name) && !t.passed,
).length;
const totalCount = result.cases.length;
const passedCount = totalCount - failedCount - expectedFailedAndFailedCount;
return {
failed,
failedCount,
passedCount,
totalCount,
expectedFailedButPassed,
expectedFailedButPassedCount,
expectedFailedAndFailedCount,
};
}
function reportVariation(result: TestResult, expectation: boolean | string[]) {
if (result.status !== 0) {
console.log(`test stderr:`);
writeAllSync(Deno.stdout, new TextEncoder().encode(result.stderr));
const expectFail = expectation === false;
console.log(
`\nfile result: ${
expectFail ? yellow("failed (expected)") : red("failed")
}. runner failed during test\n`,
);
return;
}
const {
failed,
failedCount,
passedCount,
totalCount,
expectedFailedButPassed,
expectedFailedButPassedCount,
expectedFailedAndFailedCount,
} = analyzeTestResult(result, expectation);
if (failed.length > 0) {
console.log(`\nfailures:`);
}
for (const result of failed) {
console.log(`\n${result.name}\n${result.message}\n${result.stack}`);
}
if (failed.length > 0) {
console.log(`\nfailures:\n`);
}
for (const result of failed) {
console.log(` ${JSON.stringify(result.name)}`);
}
if (expectedFailedButPassedCount > 0) {
console.log(`\nexpected failures that passed:\n`);
}
for (const result of expectedFailedButPassed) {
console.log(` ${JSON.stringify(result.name)}`);
}
console.log(
`\nfile result: ${
failedCount > 0 ? red("failed") : green("ok")
}. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`,
);
}
function createReportTestCase(expectation: boolean | string[]) {
return function reportTestCase({ name, status }: TestCaseResult) {
const expectFail = getExpectFailForCase(expectation, name);
let simpleMessage = `test ${name} ... `;
switch (status) {
case 0:
if (expectFail) {
simpleMessage += red("ok (expected fail)");
} else {
simpleMessage += green("ok");
if (quiet) {
// don't print `ok` tests if --quiet is enabled
return;
}
}
break;
case 1:
if (expectFail) {
simpleMessage += yellow("failed (expected)");
} else {
simpleMessage += red("failed");
}
break;
case 2:
if (expectFail) {
simpleMessage += yellow("failed (expected)");
} else {
simpleMessage += red("failed (timeout)");
}
break;
case 3:
if (expectFail) {
simpleMessage += yellow("failed (expected)");
} else {
simpleMessage += red("failed (incomplete)");
}
break;
}
writeAllSync(Deno.stdout, new TextEncoder().encode(simpleMessage + "\n"));
};
}
function discoverTestsToRun(
filter?: string[],
expectation: Expectation | string[] | boolean = getExpectation(),
): TestToRun[] {
const manifestFolder = getManifest().items.testharness;
const testsToRun: TestToRun[] = [];
function walk(
parentFolder: ManifestFolder,
parentExpectation: Expectation | string[] | boolean,
prefix: string,
) {
for (const key in parentFolder) {
const entry = parentFolder[key];
if (Array.isArray(entry)) {
for (
const [path, options] of entry.slice(
1,
) as ManifestTestVariation[]
) {
if (!path) continue;
const url = new URL(path, "http://web-platform.test:8000");
if (
!url.pathname.endsWith(".any.html") &&
!url.pathname.endsWith(".window.html")
) {
continue;
}
const finalPath = url.pathname + url.search;
const split = finalPath.split("/");
const finalKey = split[split.length - 1];
const expectation = Array.isArray(parentExpectation) ||
typeof parentExpectation == "boolean"
? parentExpectation
: parentExpectation[finalKey];
if (expectation === undefined) continue;
assert(
Array.isArray(expectation) || typeof expectation == "boolean",
"test entry must not have a folder expectation",
);
if (
filter &&
!filter.find((filter) => finalPath.substring(1).startsWith(filter))
) {
continue;
}
testsToRun.push({
path: finalPath,
url,
options,
expectation,
});
}
} else {
const expectation = Array.isArray(parentExpectation) ||
typeof parentExpectation == "boolean"
? parentExpectation
: parentExpectation[key];
if (expectation === undefined) continue;
walk(entry, expectation, `${prefix}/${key}`);
}
}
}
walk(manifestFolder, expectation, "");
return testsToRun;
}