tests: add web platform test runner (#8990)

Co-authored-by: Kitson Kelly <me@kitsonkelly.com>

.dprintrc.json

@@ -22,6 +22,7 @@
     "cli/dts/typescript.d.ts",
     "cli/tests/encoding",
     "cli/tsc/*typescript.js",
+    "test_util/wpt",
     "gh-pages",
     "std/**/testdata",
     "std/**/vendor",

.gitmodules

@@ -5,3 +5,6 @@
 [submodule "std/wasi/testdata"]
 	path = std/wasi/testdata
 	url = https://github.com/khronosproject/wasi-test-suite.git
+[submodule "test_util/wpt"]
+	path = test_util/wpt
+	url = https://github.com/web-platform-tests/wpt.git

cli/tests/WPT.md

@@ -0,0 +1,34 @@
+## Web Platform Tests
+
+The WPT are test suites for Web platform specs, like Fetch, WHATWG Streams, or
+console. Deno is able to run most `.any.js` and `.window.js` web platform tests.
+
+This directory contains a `wpt.json` file that is used to configure our WPT test
+runner. You can use this json file to set which WPT suites to run, and which
+tests we expect to fail (due to bugs or because they are out of scope for Deno).
+
+To include a new test file to run, add it to the array of test files for the
+corresponding suite. For example we want to enable
+`streams/readable-streams/general`. The file would then look like this:
+
+```json
+{
+  "streams": ["readable-streams/general"]
+}
+```
+
+If you need more configurability over which test cases in a test file of a suite
+to run, you can use the object representation. In the example below, we
+configure `streams/readable-streams/general` to expect
+`ReadableStream can't be constructed with an invalid type` to fail.
+
+```json
+{
+  "streams": [
+    {
+      "name": "readable-streams/general",
+      "expectFail": ["ReadableStream can't be constructed with an invalid type"]
+    }
+  ]
+}
+```

cli/tests/integration_tests.rs

@@ -5,9 +5,12 @@ use deno_core::serde_json;
 use deno_core::url;
 use deno_runtime::deno_fetch::reqwest;
 use std::io::{BufRead, Write};
+use std::path::Path;
+use std::path::PathBuf;
 use std::process::Command;
 use tempfile::TempDir;
 use test_util as util;
+use walkdir::WalkDir;
 
 macro_rules! itest(
   ($name:ident {$( $key:ident: $value:expr,)*})  => {
@@ -4915,3 +4918,171 @@ fn standalone_runtime_flags() {
   assert!(util::strip_ansi_codes(&stderr_str)
     .contains("PermissionDenied: write access"));
 }
+
+fn concat_bundle(files: Vec<(PathBuf, String)>, bundle_path: &Path) -> String {
+  let bundle_url = url::Url::from_file_path(bundle_path).unwrap().to_string();
+
+  let mut bundle = String::new();
+  let mut bundle_line_count = 0;
+  let mut source_map = sourcemap::SourceMapBuilder::new(Some(&bundle_url));
+
+  for (path, text) in files {
+    let path = std::fs::canonicalize(path).unwrap();
+    let url = url::Url::from_file_path(path).unwrap().to_string();
+    let src_id = source_map.add_source(&url);
+    source_map.set_source_contents(src_id, Some(&text));
+
+    for (line_index, line) in text.lines().enumerate() {
+      bundle.push_str(line);
+      bundle.push('\n');
+      source_map.add_raw(
+        bundle_line_count,
+        0,
+        line_index as u32,
+        0,
+        Some(src_id),
+        None,
+      );
+
+      bundle_line_count += 1;
+    }
+    bundle.push('\n');
+    bundle_line_count += 1;
+  }
+
+  let mut source_map_buf: Vec<u8> = vec![];
+  source_map
+    .into_sourcemap()
+    .to_writer(&mut source_map_buf)
+    .unwrap();
+
+  bundle.push_str("//# sourceMappingURL=data:application/json;base64,");
+  let encoded_map = base64::encode(source_map_buf);
+  bundle.push_str(&encoded_map);
+
+  bundle
+}
+
+#[test]
+fn web_platform_tests() {
+  use deno_core::serde::Deserialize;
+
+  #[derive(Deserialize)]
+  #[serde(untagged)]
+  enum WptConfig {
+    Simple(String),
+    #[serde(rename_all = "camelCase")]
+    Options {
+      name: String,
+      expect_fail: Vec<String>,
+    },
+  }
+
+  let text =
+    std::fs::read_to_string(util::tests_path().join("wpt.json")).unwrap();
+  let config: std::collections::HashMap<String, Vec<WptConfig>> =
+    deno_core::serde_json::from_str(&text).unwrap();
+
+  for (suite_name, includes) in config.into_iter() {
+    let suite_path = util::wpt_path().join(suite_name);
+    let dir = WalkDir::new(&suite_path)
+      .into_iter()
+      .filter_map(Result::ok)
+      .filter(|e| e.file_type().is_file())
+      .filter(|f| {
+        let filename = f.file_name().to_str().unwrap();
+        filename.ends_with(".any.js") || filename.ends_with(".window.js")
+      })
+      .filter_map(|f| {
+        let path = f
+          .path()
+          .strip_prefix(&suite_path)
+          .unwrap()
+          .to_str()
+          .unwrap();
+        for cfg in &includes {
+          match cfg {
+            WptConfig::Simple(name) if path.starts_with(name) => {
+              return Some((f.path().to_owned(), vec![]))
+            }
+            WptConfig::Options { name, expect_fail }
+              if path.starts_with(name) =>
+            {
+              return Some((f.path().to_owned(), expect_fail.to_vec()))
+            }
+            _ => {}
+          }
+        }
+        None
+      });
+
+    let testharness_path = util::wpt_path().join("resources/testharness.js");
+    let testharness_text = std::fs::read_to_string(&testharness_path).unwrap();
+    let testharnessreporter_path =
+      util::tests_path().join("wpt_testharnessconsolereporter.js");
+    let testharnessreporter_text =
+      std::fs::read_to_string(&testharnessreporter_path).unwrap();
+
+    for (test_file_path, expect_fail) in dir {
+      let test_file_text = std::fs::read_to_string(&test_file_path).unwrap();
+      let imports: Vec<(PathBuf, String)> = test_file_text
+        .split('\n')
+        .into_iter()
+        .filter_map(|t| t.strip_prefix("// META: script="))
+        .map(|s| {
+          let s = if s == "/resources/WebIDLParser.js" {
+            "/resources/webidl2/lib/webidl2.js"
+          } else {
+            s
+          };
+          if s.starts_with('/') {
+            util::wpt_path().join(format!(".{}", s))
+          } else if s.starts_with('.') {
+            test_file_path.parent().unwrap().join(s)
+          } else {
+            PathBuf::from(s)
+          }
+        })
+        .map(|path| {
+          let text = std::fs::read_to_string(&path).unwrap();
+          (path, text)
+        })
+        .collect();
+
+      let mut files = Vec::with_capacity(3 + imports.len());
+      files.push((testharness_path.clone(), testharness_text.clone()));
+      files.push((
+        testharnessreporter_path.clone(),
+        testharnessreporter_text.clone(),
+      ));
+      files.extend(imports);
+      files.push((test_file_path.clone(), test_file_text));
+
+      let mut file = tempfile::Builder::new()
+        .prefix("wpt-bundle-")
+        .suffix(".js")
+        .rand_bytes(5)
+        .tempfile()
+        .unwrap();
+
+      let bundle = concat_bundle(files, file.path());
+      file.write_all(bundle.as_bytes()).unwrap();
+
+      let child = util::deno_cmd()
+        .current_dir(test_file_path.parent().unwrap())
+        .arg("run")
+        .arg("-A")
+        .arg(file.path())
+        .arg(deno_core::serde_json::to_string(&expect_fail).unwrap())
+        .stdin(std::process::Stdio::piped())
+        .spawn()
+        .unwrap();
+
+      let output = child.wait_with_output().unwrap();
+      if !output.status.success() {
+        file.keep().unwrap();
+      }
+      assert!(output.status.success());
+    }
+  }
+}

cli/tests/wpt.json

@@ -0,0 +1,12 @@
+{
+  "streams": [
+    {
+      "name": "readable-streams/general",
+      "expectFail": [
+        "ReadableStream can't be constructed with an invalid type",
+        "default ReadableStream getReader() should only accept mode:undefined"
+      ]
+    },
+    "writable-streams/general"
+  ]
+}

cli/tests/wpt_testharnessconsolereporter.js

@@ -0,0 +1,119 @@
+const noColor = globalThis.Deno?.noColor ?? true;
+const enabled = !noColor;
+
+function code(open, close) {
+  return {
+    open: `\x1b[${open.join(";")}m`,
+    close: `\x1b[${close}m`,
+    regexp: new RegExp(`\\x1b\\[${close}m`, "g"),
+  };
+}
+
+function run(str, code) {
+  return enabled
+    ? `${code.open}${str.replace(code.regexp, code.open)}${code.close}`
+    : str;
+}
+
+function red(str) {
+  return run(str, code([31], 39));
+}
+
+export function green(str) {
+  return run(str, code([32], 39));
+}
+
+export function yellow(str) {
+  return run(str, code([33], 39));
+}
+
+const testResults = [];
+const testsExpectFail = JSON.parse(Deno.args[0]);
+
+window.add_result_callback(({ message, name, stack, status }) => {
+  const expectFail = testsExpectFail.includes(name);
+  let simpleMessage = `test ${name} ... `;
+  switch (status) {
+    case 0:
+      if (expectFail) {
+        simpleMessage += red("ok (expected fail)");
+      } else {
+        simpleMessage += green("ok");
+      }
+      break;
+    case 1:
+      if (expectFail) {
+        simpleMessage += yellow("failed (expected)");
+      } else {
+        simpleMessage += red("failed");
+      }
+      break;
+    case 2:
+      if (expectFail) {
+        simpleMessage += yellow("failed (expected)");
+      } else {
+        simpleMessage += red("failed (timeout)");
+      }
+      break;
+    case 3:
+      if (expectFail) {
+        simpleMessage += yellow("failed (expected)");
+      } else {
+        simpleMessage += red("failed (incomplete)");
+      }
+      break;
+  }
+
+  console.log(simpleMessage);
+
+  testResults.push({
+    name,
+    passed: status === 0,
+    expectFail,
+    message,
+    stack,
+  });
+});
+
+window.add_completion_callback((tests, harnessStatus) => {
+  const failed = testResults.filter((t) => !t.expectFail && !t.passed);
+  const expectedFailedButPassed = testResults.filter((t) =>
+    t.expectFail && t.passed
+  );
+  const expectedFailedButPassedCount = expectedFailedButPassed.length;
+  const failedCount = failed.length + expectedFailedButPassedCount;
+  const expectedFailedAndFailedCount = testResults.filter((t) =>
+    t.expectFail && !t.passed
+  ).length;
+  const totalCount = testResults.length;
+  const passedCount = totalCount - failedCount - expectedFailedAndFailedCount;
+
+  if (failed.length > 0) {
+    console.log(`\nfailures:`);
+  }
+  for (const result of failed) {
+    console.log(
+      `\n${result.name}\n${result.message}\n${result.stack}`,
+    );
+  }
+
+  if (failed.length > 0) {
+    console.log(`\nfailures:\n`);
+  }
+  for (const result of failed) {
+    console.log(`        ${result.name}`);
+  }
+  if (expectedFailedButPassedCount > 0) {
+    console.log(`\nexpected failures that passed:\n`);
+  }
+  for (const result of expectedFailedButPassed) {
+    console.log(`        ${result.name}`);
+  }
+  console.log(
+    `\ntest result: ${
+      failedCount > 0 ? red("failed") : green("ok")
+    }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`,
+  );
+
+  Deno.exit(failedCount > 0 ? 1 : 0);
+});

test_util/src/lib.rs

@@ -83,6 +83,10 @@ pub fn tests_path() -> PathBuf {
   root_path().join("cli").join("tests")
 }
 
+pub fn wpt_path() -> PathBuf {
+  root_path().join("test_util").join("wpt")
+}
+
 pub fn third_party_path() -> PathBuf {
   root_path().join("third_party")
 }
@@ -90,7 +94,6 @@ pub fn third_party_path() -> PathBuf {
 pub fn target_dir() -> PathBuf {
   let current_exe = std::env::current_exe().unwrap();
   let target_dir = current_exe.parent().unwrap().parent().unwrap();
-  println!("target_dir {}", target_dir.display());
   target_dir.into()
 }
 

test_util/wpt

@@ -0,0 +1 @@
+Subproject commit 077d53c8da8b47c1d5060893af96a29f27b10008

tools/lint.js

@@ -27,6 +27,7 @@ async function dlint() {
     ":!:cli/tests/lint/**",
     ":!:cli/tests/tsc/**",
     ":!:cli/tsc/*typescript.js",
+    ":!:test_util/wpt/**",
   ]);
 
   if (!sourceFiles.length) {