denoland-deno/tools/wpt.ts

#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.

// This script is used to run WPT tests for Deno.

import {
  runSingleTest,
  runWithTestUtil,
  TestCaseResult,
  TestResult,
} from "./wpt/runner.ts";
import {
  assert,
  autoConfig,
  cargoBuild,
  checkPy3Available,
  Expectation,
  generateRunInfo,
  getExpectation,
  getExpectFailForCase,
  getManifest,
  json,
  ManifestFolder,
  ManifestTestOptions,
  ManifestTestVariation,
  quiet,
  rest,
  runPy,
  updateManifest,
  wptreport,
} from "./wpt/utils.ts";
import { blue, bold, green, red, yellow } from "../test_util/std/fmt/colors.ts";
import { writeAll, writeAllSync } from "../test_util/std/io/util.ts";
import { saveExpectation } from "./wpt/utils.ts";

const command = Deno.args[0];

switch (command) {
  case "setup":
    await checkPy3Available();
    await updateManifest();
    await setup();
    break;

  case "run":
    await cargoBuild();
    await run();
    break;

  case "update":
    await cargoBuild();
    await update();
    break;

  default:
    console.log(`Possible commands:

    setup
      Validate that your environment is configured correctly, or help you configure it.

    run
      Run all tests like specified in \`expectation.json\`.

    update
      Update the \`expectation.json\` to match the current reality.

More details at https://deno.land/manual@main/contributing/web_platform_tests

    `);
    break;
}

async function setup() {
  const hostsPath = Deno.build.os == "windows"
    ? `${Deno.env.get("SystemRoot")}\\System32\\drivers\\etc\\hosts`
    : "/etc/hosts";
  // TODO(lucacsonato): use this when 1.7.1 is released.
  // const records = await Deno.resolveDns("web-platform.test", "A");
  // const etcHostsConfigured = records[0] == "127.0.0.1";
  const hostsFile = await Deno.readTextFile(hostsPath);
  const etcHostsConfigured = hostsFile.includes("web-platform.test");

  if (etcHostsConfigured) {
    console.log(hostsPath + " is already configured.");
  } else {
    const autoConfigure = autoConfig ||
      confirm(
        `The WPT require certain entries to be present in your ${hostsPath} file. Should these be configured automatically?`,
      );
    if (autoConfigure) {
      const proc = runPy(["wpt", "make-hosts-file"], { stdout: "piped" });
      const status = await proc.status();
      assert(status.success, "wpt make-hosts-file should not fail");
      const entries = new TextDecoder().decode(await proc.output());
      const file = await Deno.open(hostsPath, { append: true }).catch((err) => {
        if (err instanceof Deno.errors.PermissionDenied) {
          throw new Error(
            `Failed to open ${hostsPath} (permission error). Please run this command again with sudo, or configure the entries manually.`,
          );
        } else {
          throw err;
        }
      });
      await writeAll(
        file,
        new TextEncoder().encode(
          "\n\n# Configured for Web Platform Tests (Deno)\n" + entries,
        ),
      );
      console.log(`Updated ${hostsPath}`);
    } else {
      console.log(`Please configure the ${hostsPath} entries manually.`);
      if (Deno.build.os == "windows") {
        console.log("To do this run the following command in PowerShell:");
        console.log("");
        console.log("    cd test_util/wpt/");
        console.log(
          "    python.exe wpt make-hosts-file | Out-File $env:SystemRoot\\System32\\drivers\\etc\\hosts -Encoding ascii -Append",
        );
        console.log("");
      } else {
        console.log("To do this run the following command in your shell:");
        console.log("");
        console.log("    cd test_util/wpt/");
        console.log(
          "    python3 ./wpt make-hosts-file | sudo tee -a /etc/hosts",
        );
        console.log("");
      }
    }
  }

  console.log(green("Setup complete!"));
}

interface TestToRun {
  path: string;
  url: URL;
  options: ManifestTestOptions;
  expectation: boolean | string[];
}

async function run() {
  const startTime = new Date().getTime();
  assert(Array.isArray(rest), "filter must be array");
  const expectation = getExpectation();
  const tests = discoverTestsToRun(
    rest.length == 0 ? undefined : rest,
    expectation,
  );
  assertAllExpectationsHaveTests(expectation, tests, rest);
  console.log(`Going to run ${tests.length} test files.`);

  const results = await runWithTestUtil(false, async () => {
    const results = [];

    for (const test of tests) {
      console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
      const result = await runSingleTest(
        test.url,
        test.options,
        createReportTestCase(test.expectation),
      );
      results.push({ test, result });
      reportVariation(result, test.expectation);
    }

    return results;
  });
  const endTime = new Date().getTime();

  if (json) {
    const minifiedResults = [];
    for (const result of results) {
      const minified = {
        file: result.test.path,
        name:
          Object.fromEntries(result.test.options.script_metadata ?? []).title ??
            null,
        cases: result.result.cases.map((case_) => ({
          name: case_.name,
          passed: case_.passed,
        })),
      };
      minifiedResults.push(minified);
    }
    await Deno.writeTextFile(json, JSON.stringify(minifiedResults));
  }

  if (wptreport) {
    const report = await generateWptReport(results, startTime, endTime);
    await Deno.writeTextFile(wptreport, JSON.stringify(report));
  }

  const code = reportFinal(results);
  Deno.exit(code);
}

async function generateWptReport(
  results: { test: TestToRun; result: TestResult }[],
  startTime: number,
  endTime: number,
) {
  const runInfo = await generateRunInfo();
  const reportResults = [];
  for (const { test, result } of results) {
    const status = result.status !== 0
      ? "CRASH"
      : result.harnessStatus?.status === 0
      ? "OK"
      : "ERROR";
    const reportResult = {
      test: test.url.pathname + test.url.search + test.url.hash,
      subtests: result.cases.map((case_) => {
        let expected = undefined;
        if (!case_.passed) {
          if (typeof test.expectation === "boolean") {
            expected = test.expectation ? "PASS" : "FAIL";
          } else {
            expected = test.expectation.includes(case_.name) ? "FAIL" : "PASS";
          }
        }

        return {
          name: case_.name,
          status: case_.passed ? "PASS" : "FAIL",
          message: case_.message,
          expected,
          known_intermittent: [],
        };
      }),
      status,
      message: result.harnessStatus?.message ??
        (result.stderr.trim() || null),
      duration: result.duration,
      expected: status === "OK" ? undefined : "OK",
      "known_intermittent": [],
    };
    reportResults.push(reportResult);
  }
  return {
    "run_info": runInfo,
    "time_start": startTime,
    "time_end": endTime,
    "results": reportResults,
  };
}

// Check that all expectations in the expectations file have a test that will be
// run.
function assertAllExpectationsHaveTests(
  expectation: Expectation,
  testsToRun: TestToRun[],
  filter?: string[],
): void {
  const tests = new Set(testsToRun.map((t) => t.path));
  const missingTests: string[] = [];

  function walk(parentExpectation: Expectation, parent: string) {
    for (const key in parentExpectation) {
      const path = `${parent}/${key}`;
      if (
        filter &&
        !filter.find((filter) => path.substring(1).startsWith(filter))
      ) {
        continue;
      }
      const expectation = parentExpectation[key];
      if (typeof expectation == "boolean" || Array.isArray(expectation)) {
        if (!tests.has(path)) {
          missingTests.push(path);
        }
      } else {
        walk(expectation, path);
      }
    }
  }

  walk(expectation, "");

  if (missingTests.length > 0) {
    console.log(
      red(
        "Following tests are missing in manifest, but are present in expectations:",
      ),
    );
    console.log("");
    console.log(missingTests.join("\n"));
    Deno.exit(1);
  }
}

async function update() {
  assert(Array.isArray(rest), "filter must be array");
  const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
  console.log(`Going to run ${tests.length} test files.`);

  const results = await runWithTestUtil(false, async () => {
    const results = [];

    for (const test of tests) {
      console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
      const result = await runSingleTest(
        test.url,
        test.options,
        json ? () => {} : createReportTestCase(test.expectation),
      );
      results.push({ test, result });
      reportVariation(result, test.expectation);
    }

    return results;
  });

  if (json) {
    await Deno.writeTextFile(json, JSON.stringify(results));
  }

  const resultTests: Record<
    string,
    { passed: string[]; failed: string[]; status: number }
  > = {};
  for (const { test, result } of results) {
    if (!resultTests[test.path]) {
      resultTests[test.path] = {
        passed: [],
        failed: [],
        status: result.status,
      };
    }
    for (const case_ of result.cases) {
      if (case_.passed) {
        resultTests[test.path].passed.push(case_.name);
      } else {
        resultTests[test.path].failed.push(case_.name);
      }
    }
  }

  const currentExpectation = getExpectation();

  for (const path in resultTests) {
    const { passed, failed, status } = resultTests[path];
    let finalExpectation: boolean | string[];
    if (failed.length == 0 && status == 0) {
      finalExpectation = true;
    } else if (failed.length > 0 && passed.length > 0 && status == 0) {
      finalExpectation = failed;
    } else {
      finalExpectation = false;
    }

    insertExpectation(
      path.slice(1).split("/"),
      currentExpectation,
      finalExpectation,
    );
  }

  saveExpectation(currentExpectation);

  reportFinal(results);

  console.log(blue("Updated expectation.json to match reality."));

  Deno.exit(0);
}

function insertExpectation(
  segments: string[],
  currentExpectation: Expectation,
  finalExpectation: boolean | string[],
) {
  const segment = segments.shift();
  assert(segment, "segments array must never be empty");
  if (segments.length > 0) {
    if (
      !currentExpectation[segment] ||
      Array.isArray(currentExpectation[segment]) ||
      typeof currentExpectation[segment] === "boolean"
    ) {
      currentExpectation[segment] = {};
    }
    insertExpectation(
      segments,
      currentExpectation[segment] as Expectation,
      finalExpectation,
    );
  } else {
    currentExpectation[segment] = finalExpectation;
  }
}

function reportFinal(
  results: { test: TestToRun; result: TestResult }[],
): number {
  const finalTotalCount = results.length;
  let finalFailedCount = 0;
  const finalFailed: [string, TestCaseResult][] = [];
  let finalExpectedFailedAndFailedCount = 0;
  const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = [];
  const finalExpectedFailedButPassedFiles: string[] = [];
  const finalFailedFiles: string[] = [];
  for (const { test, result } of results) {
    const {
      failed,
      failedCount,
      expectedFailedButPassed,
      expectedFailedAndFailedCount,
    } = analyzeTestResult(
      result,
      test.expectation,
    );
    if (result.status !== 0) {
      if (test.expectation === false) {
        finalExpectedFailedAndFailedCount += 1;
      } else {
        finalFailedCount += 1;
        finalFailedFiles.push(test.path);
      }
    } else if (failedCount > 0) {
      finalFailedCount += 1;
      for (const case_ of failed) {
        finalFailed.push([test.path, case_]);
      }
      for (const case_ of expectedFailedButPassed) {
        finalExpectedFailedButPassedTests.push([test.path, case_]);
      }
    } else if (
      test.expectation === false &&
      expectedFailedAndFailedCount != result.cases.length
    ) {
      finalExpectedFailedButPassedFiles.push(test.path);
    }
  }
  const finalPassedCount = finalTotalCount - finalFailedCount;

  console.log(bold(blue("=".repeat(40))));

  if (finalFailed.length > 0) {
    console.log(`\nfailures:\n`);
  }
  for (const result of finalFailed) {
    console.log(
      `        ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
    );
  }
  if (finalFailedFiles.length > 0) {
    console.log(`\nfile failures:\n`);
  }
  for (const result of finalFailedFiles) {
    console.log(
      `        ${JSON.stringify(result)}`,
    );
  }
  if (finalExpectedFailedButPassedTests.length > 0) {
    console.log(`\nexpected test failures that passed:\n`);
  }
  for (const result of finalExpectedFailedButPassedTests) {
    console.log(
      `        ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
    );
  }
  if (finalExpectedFailedButPassedFiles.length > 0) {
    console.log(`\nexpected file failures that passed:\n`);
  }
  for (const result of finalExpectedFailedButPassedFiles) {
    console.log(`        ${JSON.stringify(result)}`);
  }

  const failed = (finalFailedCount > 0) ||
    (finalExpectedFailedButPassedFiles.length > 0);

  console.log(
    `\nfinal result: ${
      failed ? red("failed") : green("ok")
    }. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount}\n`,
  );

  return failed ? 1 : 0;
}

function analyzeTestResult(
  result: TestResult,
  expectation: boolean | string[],
): {
  failed: TestCaseResult[];
  failedCount: number;
  passedCount: number;
  totalCount: number;
  expectedFailedButPassed: TestCaseResult[];
  expectedFailedButPassedCount: number;
  expectedFailedAndFailedCount: number;
} {
  const failed = result.cases.filter(
    (t) => !getExpectFailForCase(expectation, t.name) && !t.passed,
  );
  const expectedFailedButPassed = result.cases.filter(
    (t) => getExpectFailForCase(expectation, t.name) && t.passed,
  );
  const expectedFailedButPassedCount = expectedFailedButPassed.length;
  const failedCount = failed.length + expectedFailedButPassedCount;
  const expectedFailedAndFailedCount = result.cases.filter(
    (t) => getExpectFailForCase(expectation, t.name) && !t.passed,
  ).length;
  const totalCount = result.cases.length;
  const passedCount = totalCount - failedCount - expectedFailedAndFailedCount;

  return {
    failed,
    failedCount,
    passedCount,
    totalCount,
    expectedFailedButPassed,
    expectedFailedButPassedCount,
    expectedFailedAndFailedCount,
  };
}

function reportVariation(result: TestResult, expectation: boolean | string[]) {
  if (result.status !== 0) {
    console.log(`test stderr:`);
    writeAllSync(Deno.stdout, new TextEncoder().encode(result.stderr));

    const expectFail = expectation === false;
    console.log(
      `\nfile result: ${
        expectFail ? yellow("failed (expected)") : red("failed")
      }. runner failed during test\n`,
    );
    return;
  }

  const {
    failed,
    failedCount,
    passedCount,
    totalCount,
    expectedFailedButPassed,
    expectedFailedButPassedCount,
    expectedFailedAndFailedCount,
  } = analyzeTestResult(result, expectation);

  if (failed.length > 0) {
    console.log(`\nfailures:`);
  }
  for (const result of failed) {
    console.log(`\n${result.name}\n${result.message}\n${result.stack}`);
  }

  if (failedCount > 0) {
    console.log(`\nfailures:\n`);
  }
  for (const result of failed) {
    console.log(`        ${JSON.stringify(result.name)}`);
  }
  if (expectedFailedButPassedCount > 0) {
    console.log(`\nexpected failures that passed:\n`);
  }
  for (const result of expectedFailedButPassed) {
    console.log(`        ${JSON.stringify(result.name)}`);
  }
  console.log(
    `\nfile result: ${
      failedCount > 0 ? red("failed") : green("ok")
    }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`,
  );
}

function createReportTestCase(expectation: boolean | string[]) {
  return function reportTestCase({ name, status }: TestCaseResult) {
    const expectFail = getExpectFailForCase(expectation, name);
    let simpleMessage = `test ${name} ... `;
    switch (status) {
      case 0:
        if (expectFail) {
          simpleMessage += red("ok (expected fail)");
        } else {
          simpleMessage += green("ok");
          if (quiet) {
            // don't print `ok` tests if --quiet is enabled
            return;
          }
        }
        break;
      case 1:
        if (expectFail) {
          simpleMessage += yellow("failed (expected)");
        } else {
          simpleMessage += red("failed");
        }
        break;
      case 2:
        if (expectFail) {
          simpleMessage += yellow("failed (expected)");
        } else {
          simpleMessage += red("failed (timeout)");
        }
        break;
      case 3:
        if (expectFail) {
          simpleMessage += yellow("failed (expected)");
        } else {
          simpleMessage += red("failed (incomplete)");
        }
        break;
    }

    writeAllSync(Deno.stdout, new TextEncoder().encode(simpleMessage + "\n"));
  };
}

function discoverTestsToRun(
  filter?: string[],
  expectation: Expectation | string[] | boolean = getExpectation(),
): TestToRun[] {
  const manifestFolder = getManifest().items.testharness;

  const testsToRun: TestToRun[] = [];

  function walk(
    parentFolder: ManifestFolder,
    parentExpectation: Expectation | string[] | boolean,
    prefix: string,
  ) {
    for (const key in parentFolder) {
      const entry = parentFolder[key];

      if (Array.isArray(entry)) {
        for (
          const [path, options] of entry.slice(
            1,
          ) as ManifestTestVariation[]
        ) {
          if (!path) continue;
          const url = new URL(path, "http://web-platform.test:8000");
          if (
            !url.pathname.endsWith(".any.html") &&
            !url.pathname.endsWith(".window.html")
          ) {
            continue;
          }
          const finalPath = url.pathname + url.search;

          const split = finalPath.split("/");
          const finalKey = split[split.length - 1];

          const expectation = Array.isArray(parentExpectation) ||
              typeof parentExpectation == "boolean"
            ? parentExpectation
            : parentExpectation[finalKey];

          if (expectation === undefined) continue;

          assert(
            Array.isArray(expectation) || typeof expectation == "boolean",
            "test entry must not have a folder expectation",
          );

          if (
            filter &&
            !filter.find((filter) => finalPath.substring(1).startsWith(filter))
          ) {
            continue;
          }
          testsToRun.push({
            path: finalPath,
            url,
            options,
            expectation,
          });
        }
      } else {
        const expectation = Array.isArray(parentExpectation) ||
            typeof parentExpectation == "boolean"
          ? parentExpectation
          : parentExpectation[key];

        if (expectation === undefined) continue;

        walk(entry, expectation, `${prefix}/${key}`);
      }
    }
  }
  walk(manifestFolder, expectation, "");

  return testsToRun;
}