mirror of
https://github.com/denoland/deno.git
synced 2024-11-29 16:30:56 -05:00
533 lines
15 KiB
TypeScript
Executable file
533 lines
15 KiB
TypeScript
Executable file
#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run
|
|
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
|
|
|
|
// This script is used to run WPT tests for Deno.
|
|
|
|
import {
|
|
runSingleTest,
|
|
runWithTestUtil,
|
|
TestCaseResult,
|
|
TestResult,
|
|
} from "./wpt/runner.ts";
|
|
import {
|
|
assert,
|
|
autoConfig,
|
|
cargoBuild,
|
|
checkPy3Available,
|
|
Expectation,
|
|
getExpectation,
|
|
getExpectFailForCase,
|
|
getManifest,
|
|
json,
|
|
ManifestFolder,
|
|
ManifestTestOptions,
|
|
ManifestTestVariation,
|
|
quiet,
|
|
rest,
|
|
runPy,
|
|
updateManifest,
|
|
} from "./wpt/utils.ts";
|
|
import {
|
|
blue,
|
|
bold,
|
|
green,
|
|
red,
|
|
yellow,
|
|
} from "https://deno.land/std@0.84.0/fmt/colors.ts";
|
|
import { saveExpectation } from "./wpt/utils.ts";
|
|
|
|
const command = Deno.args[0];
|
|
|
|
switch (command) {
|
|
case "setup":
|
|
await checkPy3Available();
|
|
await updateManifest();
|
|
await setup();
|
|
break;
|
|
|
|
case "run":
|
|
await cargoBuild();
|
|
await run();
|
|
break;
|
|
|
|
case "update":
|
|
await cargoBuild();
|
|
await update();
|
|
break;
|
|
|
|
default:
|
|
console.log(`Possible commands:
|
|
|
|
setup
|
|
Validate that your environment is configured correctly, or help you configure it.
|
|
|
|
run
|
|
Run all tests like specified in \`expectation.json\`.
|
|
|
|
update
|
|
Update the \`expectation.json\` to match the current reality.
|
|
|
|
More details at https://deno.land/manual@main/contributing/web_platform_tests
|
|
|
|
`);
|
|
break;
|
|
}
|
|
|
|
async function setup() {
|
|
// TODO(lucacsonato): use this when 1.7.1 is released.
|
|
// const records = await Deno.resolveDns("web-platform.test", "A");
|
|
// const etcHostsConfigured = records[0] == "127.0.0.1";
|
|
const hostsFile = await Deno.readTextFile("/etc/hosts");
|
|
const etcHostsConfigured = hostsFile.includes("web-platform.test");
|
|
|
|
if (etcHostsConfigured) {
|
|
console.log("/etc/hosts is already configured.");
|
|
} else {
|
|
const autoConfigure = autoConfig ||
|
|
confirm(
|
|
"The WPT require certain entries to be present in your /etc/hosts file. Should these be configured automatically?",
|
|
);
|
|
if (autoConfigure) {
|
|
const proc = runPy(["wpt", "make-hosts-file"], { stdout: "piped" });
|
|
const status = await proc.status();
|
|
assert(status.success, "wpt make-hosts-file should not fail");
|
|
const entries = new TextDecoder().decode(await proc.output());
|
|
const hostsPath = Deno.build.os == "windows"
|
|
? `${Deno.env.get("SystemRoot")}\\System32\\drivers\\etc\\hosts`
|
|
: "/etc/hosts";
|
|
const file = await Deno.open(hostsPath, { append: true }).catch((err) => {
|
|
if (err instanceof Deno.errors.PermissionDenied) {
|
|
throw new Error(
|
|
`Failed to open ${hostsPath} (permission error). Please run this command again with sudo, or configure the entries manually.`,
|
|
);
|
|
} else {
|
|
throw err;
|
|
}
|
|
});
|
|
await Deno.writeAll(
|
|
file,
|
|
new TextEncoder().encode(
|
|
"\n\n# Configured for Web Platform Tests (Deno)\n" + entries,
|
|
),
|
|
);
|
|
console.log("Updated /etc/hosts");
|
|
} else {
|
|
console.log("Please configure the /etc/hosts entries manually.");
|
|
if (Deno.build.os == "windows") {
|
|
console.log("To do this run the following command in PowerShell:");
|
|
console.log("");
|
|
console.log(" cd test_util/wpt/");
|
|
console.log(
|
|
" python.exe wpt make-hosts-file | Out-File $env:SystemRoot\\System32\\drivers\\etc\\hosts -Encoding ascii -Append",
|
|
);
|
|
console.log("");
|
|
} else {
|
|
console.log("To do this run the following command in your shell:");
|
|
console.log("");
|
|
console.log(" cd test_util/wpt/");
|
|
console.log(
|
|
" python3 ./wpt make-hosts-file | sudo tee -a /etc/hosts",
|
|
);
|
|
console.log("");
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(green("Setup complete!"));
|
|
}
|
|
|
|
interface TestToRun {
|
|
sourcePath: string;
|
|
path: string;
|
|
url: URL;
|
|
options: ManifestTestOptions;
|
|
expectation: boolean | string[];
|
|
}
|
|
|
|
async function run() {
|
|
assert(Array.isArray(rest), "filter must be array");
|
|
const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest);
|
|
console.log(`Going to run ${tests.length} test files.`);
|
|
|
|
const results = await runWithTestUtil(false, async () => {
|
|
const results = [];
|
|
|
|
for (const test of tests) {
|
|
console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
|
|
const result = await runSingleTest(
|
|
test.url,
|
|
test.options,
|
|
json ? () => {} : createReportTestCase(test.expectation),
|
|
);
|
|
results.push({ test, result });
|
|
reportVariation(result, test.expectation);
|
|
}
|
|
|
|
return results;
|
|
});
|
|
|
|
if (json) {
|
|
await Deno.writeTextFile(json, JSON.stringify(results));
|
|
}
|
|
const code = reportFinal(results);
|
|
Deno.exit(code);
|
|
}
|
|
|
|
async function update() {
|
|
assert(Array.isArray(rest), "filter must be array");
|
|
const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
|
|
console.log(`Going to run ${tests.length} test files.`);
|
|
|
|
const results = await runWithTestUtil(false, async () => {
|
|
const results = [];
|
|
|
|
for (const test of tests) {
|
|
console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
|
|
const result = await runSingleTest(
|
|
test.url,
|
|
test.options,
|
|
json ? () => {} : createReportTestCase(test.expectation),
|
|
);
|
|
results.push({ test, result });
|
|
reportVariation(result, test.expectation);
|
|
}
|
|
|
|
return results;
|
|
});
|
|
|
|
if (json) {
|
|
await Deno.writeTextFile(json, JSON.stringify(results));
|
|
}
|
|
|
|
const resultTests: Record<
|
|
string,
|
|
{ passed: string[]; failed: string[]; status: number }
|
|
> = {};
|
|
for (const { test, result } of results) {
|
|
if (!resultTests[test.sourcePath]) {
|
|
resultTests[test.sourcePath] = {
|
|
passed: [],
|
|
failed: [],
|
|
status: result.status,
|
|
};
|
|
}
|
|
for (const case_ of result.cases) {
|
|
if (case_.passed) {
|
|
resultTests[test.sourcePath].passed.push(case_.name);
|
|
} else {
|
|
resultTests[test.sourcePath].failed.push(case_.name);
|
|
}
|
|
}
|
|
}
|
|
|
|
const currentExpectation = getExpectation();
|
|
|
|
for (const path in resultTests) {
|
|
const { passed, failed, status } = resultTests[path];
|
|
let finalExpectation: boolean | string[];
|
|
if (failed.length == 0 && status == 0) {
|
|
finalExpectation = true;
|
|
} else if (failed.length > 0 && passed.length > 0 && status == 0) {
|
|
finalExpectation = failed;
|
|
} else {
|
|
finalExpectation = false;
|
|
}
|
|
|
|
insertExpectation(
|
|
path.slice(1).split("/"),
|
|
currentExpectation,
|
|
finalExpectation,
|
|
);
|
|
}
|
|
|
|
saveExpectation(currentExpectation);
|
|
|
|
reportFinal(results);
|
|
|
|
console.log(blue("Updated expectation.json to match reality."));
|
|
|
|
Deno.exit(0);
|
|
}
|
|
|
|
function insertExpectation(
|
|
segments: string[],
|
|
currentExpectation: Expectation,
|
|
finalExpectation: boolean | string[],
|
|
) {
|
|
const segment = segments.shift();
|
|
assert(segment, "segments array must never be empty");
|
|
if (segments.length > 0) {
|
|
if (
|
|
!currentExpectation[segment] ||
|
|
Array.isArray(currentExpectation[segment]) ||
|
|
typeof currentExpectation[segment] === "boolean"
|
|
) {
|
|
currentExpectation[segment] = {};
|
|
}
|
|
insertExpectation(
|
|
segments,
|
|
currentExpectation[segment] as Expectation,
|
|
finalExpectation,
|
|
);
|
|
} else {
|
|
currentExpectation[segment] = finalExpectation;
|
|
}
|
|
}
|
|
|
|
function reportFinal(
|
|
results: { test: TestToRun; result: TestResult }[],
|
|
): number {
|
|
const finalTotalCount = results.length;
|
|
let finalFailedCount = 0;
|
|
const finalFailed: [string, TestCaseResult][] = [];
|
|
let finalExpectedFailedAndFailedCount = 0;
|
|
const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = [];
|
|
const finalExpectedFailedButPassedFiles: string[] = [];
|
|
for (const { test, result } of results) {
|
|
const { failed, failedCount, expectedFailedButPassed } = analyzeTestResult(
|
|
result,
|
|
test.expectation,
|
|
);
|
|
if (result.status !== 0) {
|
|
if (test.expectation === false) {
|
|
finalExpectedFailedAndFailedCount += 1;
|
|
} else {
|
|
finalFailedCount += 1;
|
|
finalExpectedFailedButPassedFiles.push(test.path);
|
|
}
|
|
} else if (failedCount > 0) {
|
|
finalFailedCount += 1;
|
|
for (const case_ of failed) {
|
|
finalFailed.push([test.path, case_]);
|
|
}
|
|
for (const case_ of expectedFailedButPassed) {
|
|
finalExpectedFailedButPassedTests.push([test.path, case_]);
|
|
}
|
|
}
|
|
}
|
|
const finalPassedCount = finalTotalCount - finalFailedCount;
|
|
|
|
console.log(bold(blue("=".repeat(40))));
|
|
|
|
if (finalFailed.length > 0) {
|
|
console.log(`\nfailures:\n`);
|
|
}
|
|
for (const result of finalFailed) {
|
|
console.log(
|
|
` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
|
|
);
|
|
}
|
|
if (finalExpectedFailedButPassedTests.length > 0) {
|
|
console.log(`\nexpected test failures that passed:\n`);
|
|
}
|
|
for (const result of finalExpectedFailedButPassedTests) {
|
|
console.log(
|
|
` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
|
|
);
|
|
}
|
|
if (finalExpectedFailedButPassedFiles.length > 0) {
|
|
console.log(`\nexpected file failures that passed:\n`);
|
|
}
|
|
for (const result of finalExpectedFailedButPassedFiles) {
|
|
console.log(` ${JSON.stringify(result)}`);
|
|
}
|
|
|
|
console.log(
|
|
`\nfinal result: ${
|
|
finalFailedCount > 0 ? red("failed") : green("ok")
|
|
}. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount}\n`,
|
|
);
|
|
|
|
return finalFailedCount > 0 ? 1 : 0;
|
|
}
|
|
|
|
function analyzeTestResult(
|
|
result: TestResult,
|
|
expectation: boolean | string[],
|
|
): {
|
|
failed: TestCaseResult[];
|
|
failedCount: number;
|
|
passedCount: number;
|
|
totalCount: number;
|
|
expectedFailedButPassed: TestCaseResult[];
|
|
expectedFailedButPassedCount: number;
|
|
expectedFailedAndFailedCount: number;
|
|
} {
|
|
const failed = result.cases.filter(
|
|
(t) => !getExpectFailForCase(expectation, t.name) && !t.passed,
|
|
);
|
|
const expectedFailedButPassed = result.cases.filter(
|
|
(t) => getExpectFailForCase(expectation, t.name) && t.passed,
|
|
);
|
|
const expectedFailedButPassedCount = expectedFailedButPassed.length;
|
|
const failedCount = failed.length + expectedFailedButPassedCount;
|
|
const expectedFailedAndFailedCount = result.cases.filter(
|
|
(t) => getExpectFailForCase(expectation, t.name) && !t.passed,
|
|
).length;
|
|
const totalCount = result.cases.length;
|
|
const passedCount = totalCount - failedCount - expectedFailedAndFailedCount;
|
|
|
|
return {
|
|
failed,
|
|
failedCount,
|
|
passedCount,
|
|
totalCount,
|
|
expectedFailedButPassed,
|
|
expectedFailedButPassedCount,
|
|
expectedFailedAndFailedCount,
|
|
};
|
|
}
|
|
|
|
function reportVariation(result: TestResult, expectation: boolean | string[]) {
|
|
if (result.status !== 0) {
|
|
console.log(`test stderr:`);
|
|
Deno.writeAllSync(Deno.stdout, new TextEncoder().encode(result.stderr));
|
|
|
|
const expectFail = expectation === false;
|
|
console.log(
|
|
`\nfile result: ${
|
|
expectFail ? yellow("failed (expected)") : red("failed")
|
|
}. runner failed during test\n`,
|
|
);
|
|
return;
|
|
}
|
|
|
|
const {
|
|
failed,
|
|
failedCount,
|
|
passedCount,
|
|
totalCount,
|
|
expectedFailedButPassed,
|
|
expectedFailedButPassedCount,
|
|
expectedFailedAndFailedCount,
|
|
} = analyzeTestResult(result, expectation);
|
|
|
|
if (failed.length > 0) {
|
|
console.log(`\nfailures:`);
|
|
}
|
|
for (const result of failed) {
|
|
console.log(`\n${result.name}\n${result.message}\n${result.stack}`);
|
|
}
|
|
|
|
if (failed.length > 0) {
|
|
console.log(`\nfailures:\n`);
|
|
}
|
|
for (const result of failed) {
|
|
console.log(` ${JSON.stringify(result.name)}`);
|
|
}
|
|
if (expectedFailedButPassedCount > 0) {
|
|
console.log(`\nexpected failures that passed:\n`);
|
|
}
|
|
for (const result of expectedFailedButPassed) {
|
|
console.log(` ${JSON.stringify(result.name)}`);
|
|
}
|
|
console.log(
|
|
`\nfile result: ${
|
|
failedCount > 0 ? red("failed") : green("ok")
|
|
}. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`,
|
|
);
|
|
}
|
|
|
|
function createReportTestCase(expectation: boolean | string[]) {
|
|
return function reportTestCase({ name, status }: TestCaseResult) {
|
|
const expectFail = getExpectFailForCase(expectation, name);
|
|
let simpleMessage = `test ${name} ... `;
|
|
switch (status) {
|
|
case 0:
|
|
if (expectFail) {
|
|
simpleMessage += red("ok (expected fail)");
|
|
} else {
|
|
simpleMessage += green("ok");
|
|
if (quiet) {
|
|
// don't print `ok` tests if --quiet is enabled
|
|
return;
|
|
}
|
|
}
|
|
break;
|
|
case 1:
|
|
if (expectFail) {
|
|
simpleMessage += yellow("failed (expected)");
|
|
} else {
|
|
simpleMessage += red("failed");
|
|
}
|
|
break;
|
|
case 2:
|
|
if (expectFail) {
|
|
simpleMessage += yellow("failed (expected)");
|
|
} else {
|
|
simpleMessage += red("failed (timeout)");
|
|
}
|
|
break;
|
|
case 3:
|
|
if (expectFail) {
|
|
simpleMessage += yellow("failed (expected)");
|
|
} else {
|
|
simpleMessage += red("failed (incomplete)");
|
|
}
|
|
break;
|
|
}
|
|
|
|
console.log(simpleMessage);
|
|
};
|
|
}
|
|
|
|
function discoverTestsToRun(
|
|
filter?: string[],
|
|
expectation: Expectation | string[] | boolean = getExpectation(),
|
|
): TestToRun[] {
|
|
const manifestFolder = getManifest().items.testharness;
|
|
|
|
const testsToRun: TestToRun[] = [];
|
|
|
|
function walk(
|
|
parentFolder: ManifestFolder,
|
|
parentExpectation: Expectation | string[] | boolean,
|
|
prefix: string,
|
|
) {
|
|
for (const key in parentFolder) {
|
|
const sourcePath = `${prefix}/${key}`;
|
|
const entry = parentFolder[key];
|
|
const expectation = Array.isArray(parentExpectation) ||
|
|
typeof parentExpectation == "boolean"
|
|
? parentExpectation
|
|
: parentExpectation[key];
|
|
|
|
if (expectation === undefined) continue;
|
|
|
|
if (Array.isArray(entry)) {
|
|
assert(
|
|
Array.isArray(expectation) || typeof expectation == "boolean",
|
|
"test entry must not have a folder expectation",
|
|
);
|
|
if (
|
|
filter &&
|
|
!filter.find((filter) => sourcePath.substring(1).startsWith(filter))
|
|
) {
|
|
continue;
|
|
}
|
|
|
|
for (
|
|
const [path, options] of entry.slice(
|
|
1,
|
|
) as ManifestTestVariation[]
|
|
) {
|
|
if (!path) continue;
|
|
const url = new URL(path, "http://web-platform.test:8000");
|
|
if (!url.pathname.endsWith(".any.html")) continue;
|
|
testsToRun.push({
|
|
sourcePath,
|
|
path: url.pathname + url.search,
|
|
url,
|
|
options,
|
|
expectation,
|
|
});
|
|
}
|
|
} else {
|
|
walk(entry, expectation, sourcePath);
|
|
}
|
|
}
|
|
}
|
|
walk(manifestFolder, expectation, "");
|
|
|
|
return testsToRun;
|
|
}
|