From a3099798c881ac1be7108c0255e67e182c7080da Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Tue, 5 Jan 2021 12:07:27 +0100 Subject: [PATCH] tests: add web platform test runner (#8990) Co-authored-by: Kitson Kelly --- .dprintrc.json | 1 + .gitmodules | 3 + cli/tests/WPT.md | 34 ++++ cli/tests/integration_tests.rs | 171 ++++++++++++++++++++ cli/tests/wpt.json | 12 ++ cli/tests/wpt_testharnessconsolereporter.js | 119 ++++++++++++++ test_util/src/lib.rs | 5 +- test_util/wpt | 1 + tools/lint.js | 1 + 9 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 cli/tests/WPT.md create mode 100644 cli/tests/wpt.json create mode 100644 cli/tests/wpt_testharnessconsolereporter.js create mode 160000 test_util/wpt diff --git a/.dprintrc.json b/.dprintrc.json index 9db9bca1a5..9217366411 100644 --- a/.dprintrc.json +++ b/.dprintrc.json @@ -22,6 +22,7 @@ "cli/dts/typescript.d.ts", "cli/tests/encoding", "cli/tsc/*typescript.js", + "test_util/wpt", "gh-pages", "std/**/testdata", "std/**/vendor", diff --git a/.gitmodules b/.gitmodules index cd52dfda31..22cc5436a9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -5,3 +5,6 @@ [submodule "std/wasi/testdata"] path = std/wasi/testdata url = https://github.com/khronosproject/wasi-test-suite.git +[submodule "test_util/wpt"] + path = test_util/wpt + url = https://github.com/web-platform-tests/wpt.git diff --git a/cli/tests/WPT.md b/cli/tests/WPT.md new file mode 100644 index 0000000000..553fe3263f --- /dev/null +++ b/cli/tests/WPT.md @@ -0,0 +1,34 @@ +## Web Platform Tests + +The WPT are test suites for Web platform specs, like Fetch, WHATWG Streams, or +console. Deno is able to run most `.any.js` and `.window.js` web platform tests. + +This directory contains a `wpt.json` file that is used to configure our WPT test +runner. You can use this json file to set which WPT suites to run, and which +tests we expect to fail (due to bugs or because they are out of scope for Deno). + +To include a new test file to run, add it to the array of test files for the +corresponding suite. For example we want to enable +`streams/readable-streams/general`. The file would then look like this: + +```json +{ + "streams": ["readable-streams/general"] +} +``` + +If you need more configurability over which test cases in a test file of a suite +to run, you can use the object representation. In the example below, we +configure `streams/readable-streams/general` to expect +`ReadableStream can't be constructed with an invalid type` to fail. + +```json +{ + "streams": [ + { + "name": "readable-streams/general", + "expectFail": ["ReadableStream can't be constructed with an invalid type"] + } + ] +} +``` diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index dd76c5782d..5d08ec3b66 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -5,9 +5,12 @@ use deno_core::serde_json; use deno_core::url; use deno_runtime::deno_fetch::reqwest; use std::io::{BufRead, Write}; +use std::path::Path; +use std::path::PathBuf; use std::process::Command; use tempfile::TempDir; use test_util as util; +use walkdir::WalkDir; macro_rules! itest( ($name:ident {$( $key:ident: $value:expr,)*}) => { @@ -4915,3 +4918,171 @@ fn standalone_runtime_flags() { assert!(util::strip_ansi_codes(&stderr_str) .contains("PermissionDenied: write access")); } + +fn concat_bundle(files: Vec<(PathBuf, String)>, bundle_path: &Path) -> String { + let bundle_url = url::Url::from_file_path(bundle_path).unwrap().to_string(); + + let mut bundle = String::new(); + let mut bundle_line_count = 0; + let mut source_map = sourcemap::SourceMapBuilder::new(Some(&bundle_url)); + + for (path, text) in files { + let path = std::fs::canonicalize(path).unwrap(); + let url = url::Url::from_file_path(path).unwrap().to_string(); + let src_id = source_map.add_source(&url); + source_map.set_source_contents(src_id, Some(&text)); + + for (line_index, line) in text.lines().enumerate() { + bundle.push_str(line); + bundle.push('\n'); + source_map.add_raw( + bundle_line_count, + 0, + line_index as u32, + 0, + Some(src_id), + None, + ); + + bundle_line_count += 1; + } + bundle.push('\n'); + bundle_line_count += 1; + } + + let mut source_map_buf: Vec = vec![]; + source_map + .into_sourcemap() + .to_writer(&mut source_map_buf) + .unwrap(); + + bundle.push_str("//# sourceMappingURL=data:application/json;base64,"); + let encoded_map = base64::encode(source_map_buf); + bundle.push_str(&encoded_map); + + bundle +} + +#[test] +fn web_platform_tests() { + use deno_core::serde::Deserialize; + + #[derive(Deserialize)] + #[serde(untagged)] + enum WptConfig { + Simple(String), + #[serde(rename_all = "camelCase")] + Options { + name: String, + expect_fail: Vec, + }, + } + + let text = + std::fs::read_to_string(util::tests_path().join("wpt.json")).unwrap(); + let config: std::collections::HashMap> = + deno_core::serde_json::from_str(&text).unwrap(); + + for (suite_name, includes) in config.into_iter() { + let suite_path = util::wpt_path().join(suite_name); + let dir = WalkDir::new(&suite_path) + .into_iter() + .filter_map(Result::ok) + .filter(|e| e.file_type().is_file()) + .filter(|f| { + let filename = f.file_name().to_str().unwrap(); + filename.ends_with(".any.js") || filename.ends_with(".window.js") + }) + .filter_map(|f| { + let path = f + .path() + .strip_prefix(&suite_path) + .unwrap() + .to_str() + .unwrap(); + for cfg in &includes { + match cfg { + WptConfig::Simple(name) if path.starts_with(name) => { + return Some((f.path().to_owned(), vec![])) + } + WptConfig::Options { name, expect_fail } + if path.starts_with(name) => + { + return Some((f.path().to_owned(), expect_fail.to_vec())) + } + _ => {} + } + } + None + }); + + let testharness_path = util::wpt_path().join("resources/testharness.js"); + let testharness_text = std::fs::read_to_string(&testharness_path).unwrap(); + let testharnessreporter_path = + util::tests_path().join("wpt_testharnessconsolereporter.js"); + let testharnessreporter_text = + std::fs::read_to_string(&testharnessreporter_path).unwrap(); + + for (test_file_path, expect_fail) in dir { + let test_file_text = std::fs::read_to_string(&test_file_path).unwrap(); + let imports: Vec<(PathBuf, String)> = test_file_text + .split('\n') + .into_iter() + .filter_map(|t| t.strip_prefix("// META: script=")) + .map(|s| { + let s = if s == "/resources/WebIDLParser.js" { + "/resources/webidl2/lib/webidl2.js" + } else { + s + }; + if s.starts_with('/') { + util::wpt_path().join(format!(".{}", s)) + } else if s.starts_with('.') { + test_file_path.parent().unwrap().join(s) + } else { + PathBuf::from(s) + } + }) + .map(|path| { + let text = std::fs::read_to_string(&path).unwrap(); + (path, text) + }) + .collect(); + + let mut files = Vec::with_capacity(3 + imports.len()); + files.push((testharness_path.clone(), testharness_text.clone())); + files.push(( + testharnessreporter_path.clone(), + testharnessreporter_text.clone(), + )); + files.extend(imports); + files.push((test_file_path.clone(), test_file_text)); + + let mut file = tempfile::Builder::new() + .prefix("wpt-bundle-") + .suffix(".js") + .rand_bytes(5) + .tempfile() + .unwrap(); + + let bundle = concat_bundle(files, file.path()); + file.write_all(bundle.as_bytes()).unwrap(); + + let child = util::deno_cmd() + .current_dir(test_file_path.parent().unwrap()) + .arg("run") + .arg("-A") + .arg(file.path()) + .arg(deno_core::serde_json::to_string(&expect_fail).unwrap()) + .stdin(std::process::Stdio::piped()) + .spawn() + .unwrap(); + + let output = child.wait_with_output().unwrap(); + if !output.status.success() { + file.keep().unwrap(); + } + assert!(output.status.success()); + } + } +} diff --git a/cli/tests/wpt.json b/cli/tests/wpt.json new file mode 100644 index 0000000000..013c8e6017 --- /dev/null +++ b/cli/tests/wpt.json @@ -0,0 +1,12 @@ +{ + "streams": [ + { + "name": "readable-streams/general", + "expectFail": [ + "ReadableStream can't be constructed with an invalid type", + "default ReadableStream getReader() should only accept mode:undefined" + ] + }, + "writable-streams/general" + ] +} diff --git a/cli/tests/wpt_testharnessconsolereporter.js b/cli/tests/wpt_testharnessconsolereporter.js new file mode 100644 index 0000000000..9e34d0689a --- /dev/null +++ b/cli/tests/wpt_testharnessconsolereporter.js @@ -0,0 +1,119 @@ +const noColor = globalThis.Deno?.noColor ?? true; +const enabled = !noColor; + +function code(open, close) { + return { + open: `\x1b[${open.join(";")}m`, + close: `\x1b[${close}m`, + regexp: new RegExp(`\\x1b\\[${close}m`, "g"), + }; +} + +function run(str, code) { + return enabled + ? `${code.open}${str.replace(code.regexp, code.open)}${code.close}` + : str; +} + +function red(str) { + return run(str, code([31], 39)); +} + +export function green(str) { + return run(str, code([32], 39)); +} + +export function yellow(str) { + return run(str, code([33], 39)); +} + +const testResults = []; +const testsExpectFail = JSON.parse(Deno.args[0]); + +window.add_result_callback(({ message, name, stack, status }) => { + const expectFail = testsExpectFail.includes(name); + let simpleMessage = `test ${name} ... `; + switch (status) { + case 0: + if (expectFail) { + simpleMessage += red("ok (expected fail)"); + } else { + simpleMessage += green("ok"); + } + break; + case 1: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed"); + } + break; + case 2: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed (timeout)"); + } + break; + case 3: + if (expectFail) { + simpleMessage += yellow("failed (expected)"); + } else { + simpleMessage += red("failed (incomplete)"); + } + break; + } + + console.log(simpleMessage); + + testResults.push({ + name, + passed: status === 0, + expectFail, + message, + stack, + }); +}); + +window.add_completion_callback((tests, harnessStatus) => { + const failed = testResults.filter((t) => !t.expectFail && !t.passed); + const expectedFailedButPassed = testResults.filter((t) => + t.expectFail && t.passed + ); + const expectedFailedButPassedCount = expectedFailedButPassed.length; + const failedCount = failed.length + expectedFailedButPassedCount; + const expectedFailedAndFailedCount = testResults.filter((t) => + t.expectFail && !t.passed + ).length; + const totalCount = testResults.length; + const passedCount = totalCount - failedCount - expectedFailedAndFailedCount; + + if (failed.length > 0) { + console.log(`\nfailures:`); + } + for (const result of failed) { + console.log( + `\n${result.name}\n${result.message}\n${result.stack}`, + ); + } + + if (failed.length > 0) { + console.log(`\nfailures:\n`); + } + for (const result of failed) { + console.log(` ${result.name}`); + } + if (expectedFailedButPassedCount > 0) { + console.log(`\nexpected failures that passed:\n`); + } + for (const result of expectedFailedButPassed) { + console.log(` ${result.name}`); + } + console.log( + `\ntest result: ${ + failedCount > 0 ? red("failed") : green("ok") + }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`, + ); + + Deno.exit(failedCount > 0 ? 1 : 0); +}); diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index a91416bb57..262084af72 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -83,6 +83,10 @@ pub fn tests_path() -> PathBuf { root_path().join("cli").join("tests") } +pub fn wpt_path() -> PathBuf { + root_path().join("test_util").join("wpt") +} + pub fn third_party_path() -> PathBuf { root_path().join("third_party") } @@ -90,7 +94,6 @@ pub fn third_party_path() -> PathBuf { pub fn target_dir() -> PathBuf { let current_exe = std::env::current_exe().unwrap(); let target_dir = current_exe.parent().unwrap().parent().unwrap(); - println!("target_dir {}", target_dir.display()); target_dir.into() } diff --git a/test_util/wpt b/test_util/wpt new file mode 160000 index 0000000000..077d53c8da --- /dev/null +++ b/test_util/wpt @@ -0,0 +1 @@ +Subproject commit 077d53c8da8b47c1d5060893af96a29f27b10008 diff --git a/tools/lint.js b/tools/lint.js index 18de2aef30..3f30a7915e 100755 --- a/tools/lint.js +++ b/tools/lint.js @@ -27,6 +27,7 @@ async function dlint() { ":!:cli/tests/lint/**", ":!:cli/tests/tsc/**", ":!:cli/tsc/*typescript.js", + ":!:test_util/wpt/**", ]); if (!sourceFiles.length) {