From e8258e0210c4690a1fbbcefe0e6a859da8efc19b Mon Sep 17 00:00:00 2001 From: Casper Beyer Date: Tue, 6 Jul 2021 09:20:33 +0800 Subject: [PATCH] feat(test): add --shuffle flag to randomize test ordering (#11163) --- Cargo.lock | 15 +++++------ cli/Cargo.toml | 1 + cli/flags.rs | 29 ++++++++++++++++++++- cli/main.rs | 5 ++++ cli/tests/integration/test_tests.rs | 12 +++++++++ cli/tests/test/shuffle.out | 39 +++++++++++++++++++++++++++++ cli/tests/test/shuffle/bar_test.ts | 3 +++ cli/tests/test/shuffle/baz_test.ts | 3 +++ cli/tests/test/shuffle/foo_test.ts | 3 +++ cli/tools/test_runner.rs | 15 +++++++++++ runtime/js/40_testing.js | 19 ++++++++++++++ 11 files changed, 136 insertions(+), 8 deletions(-) create mode 100644 cli/tests/test/shuffle.out create mode 100644 cli/tests/test/shuffle/bar_test.ts create mode 100644 cli/tests/test/shuffle/baz_test.ts create mode 100644 cli/tests/test/shuffle/foo_test.ts diff --git a/Cargo.lock b/Cargo.lock index d5adc09124..4ffb40c285 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -568,6 +568,7 @@ dependencies = [ "os_pipe", "percent-encoding", "pin-project", + "rand 0.8.4", "regex", "ring", "rustyline", @@ -645,7 +646,7 @@ version = "0.24.1" dependencies = [ "deno_core", "deno_web", - "rand 0.8.3", + "rand 0.8.4", "ring", "tokio", "uuid", @@ -2529,9 +2530,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", "rand_chacha 0.3.0", @@ -3553,7 +3554,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi 0.3.9", @@ -3831,7 +3832,7 @@ dependencies = [ "lazy_static", "log", "radix_trie", - "rand 0.8.3", + "rand 0.8.4", "thiserror", "tokio", "trust-dns-proto", @@ -3854,7 +3855,7 @@ dependencies = [ "ipnet", "lazy_static", "log", - "rand 0.8.3", + "rand 0.8.4", "serde", "smallvec", "thiserror", @@ -3926,7 +3927,7 @@ dependencies = [ "httparse", "input_buffer", "log", - "rand 0.8.3", + "rand 0.8.4", "rustls", "sha-1", "thiserror", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 0253693585..a4a8786a5b 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -71,6 +71,7 @@ num_cpus = "1.13.0" percent-encoding = "2.1.0" pin-project = "1.0.6" regex = "1.4.3" +rand = { version = "0.8.3", features = [ "small_rng" ] } ring = "0.16.20" rustyline = { version = "8.0.0", default-features = false } rustyline-derive = "0.4.0" diff --git a/cli/flags.rs b/cli/flags.rs index 6a51236c50..400798cbd1 100644 --- a/cli/flags.rs +++ b/cli/flags.rs @@ -103,6 +103,7 @@ pub enum DenoSubcommand { allow_none: bool, include: Option>, filter: Option, + shuffle: Option, concurrent_jobs: usize, }, Types, @@ -1016,6 +1017,20 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .help("Run tests with this string or pattern in the test name"), ) + .arg( + Arg::with_name("shuffle") + .long("shuffle") + .value_name("NUMBER") + .help("(UNSTABLE): Shuffle the order in which the tests are run") + .min_values(0) + .max_values(1) + .require_equals(true) + .takes_value(true) + .validator(|val: String| match val.parse::() { + Ok(_) => Ok(()), + Err(_) => Err("Shuffle seed should be a number".to_string()), + }), + ) .arg( Arg::with_name("coverage") .long("coverage") @@ -1686,7 +1701,17 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let quiet = matches.is_present("quiet"); let filter = matches.value_of("filter").map(String::from); - flags.watch = matches.is_present("watch"); + let shuffle = if matches.is_present("shuffle") { + let value = if let Some(value) = matches.value_of("shuffle") { + value.parse::().unwrap() + } else { + rand::random::() + }; + + Some(value) + } else { + None + }; if matches.is_present("script_arg") { let script_arg: Vec = matches @@ -1730,6 +1755,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) { quiet, include, filter, + shuffle, allow_none, concurrent_jobs, }; @@ -3366,6 +3392,7 @@ mod tests { allow_none: true, quiet: false, include: Some(svec!["dir1/", "dir2/"]), + shuffle: None, concurrent_jobs: 1, }, unstable: true, diff --git a/cli/main.rs b/cli/main.rs index 381ba7214e..a61f945301 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -984,6 +984,7 @@ async fn test_command( quiet: bool, allow_none: bool, filter: Option, + shuffle: Option, concurrent_jobs: usize, ) -> Result<(), AnyError> { if let Some(ref coverage_dir) = flags.coverage_dir { @@ -1172,6 +1173,7 @@ async fn test_command( quiet, true, filter.clone(), + shuffle, concurrent_jobs, ) .map(|res| res.map(|_| ())) @@ -1207,6 +1209,7 @@ async fn test_command( quiet, allow_none, filter, + shuffle, concurrent_jobs, ) .await?; @@ -1314,6 +1317,7 @@ fn get_subcommand( include, allow_none, filter, + shuffle, concurrent_jobs, } => test_command( flags, @@ -1324,6 +1328,7 @@ fn get_subcommand( quiet, allow_none, filter, + shuffle, concurrent_jobs, ) .boxed_local(), diff --git a/cli/tests/integration/test_tests.rs b/cli/tests/integration/test_tests.rs index 3bc81c5d30..c029b154d0 100644 --- a/cli/tests/integration/test_tests.rs +++ b/cli/tests/integration/test_tests.rs @@ -114,3 +114,15 @@ itest!(unhandled_rejection { exit_code: 1, output: "test/unhandled_rejection.out", }); + +itest!(shuffle { + args: "test --shuffle test/shuffle", + exit_code: 0, + output_str: Some("[WILDCARD]"), +}); + +itest!(shuffle_with_seed { + args: "test --shuffle=42 test/shuffle", + exit_code: 0, + output: "test/shuffle.out", +}); diff --git a/cli/tests/test/shuffle.out b/cli/tests/test/shuffle.out new file mode 100644 index 0000000000..04dd08ee20 --- /dev/null +++ b/cli/tests/test/shuffle.out @@ -0,0 +1,39 @@ +Check [WILDCARD]/test/shuffle/foo_test.ts +Check [WILDCARD]/test/shuffle/baz_test.ts +Check [WILDCARD]/test/shuffle/bar_test.ts +running 10 tests from [WILDCARD]/test/shuffle/foo_test.ts +test test 2 ... ok ([WILDCARD]) +test test 3 ... ok ([WILDCARD]) +test test 6 ... ok ([WILDCARD]) +test test 9 ... ok ([WILDCARD]) +test test 8 ... ok ([WILDCARD]) +test test 7 ... ok ([WILDCARD]) +test test 5 ... ok ([WILDCARD]) +test test 4 ... ok ([WILDCARD]) +test test 1 ... ok ([WILDCARD]) +test test 0 ... ok ([WILDCARD]) +running 10 tests from [WILDCARD]/test/shuffle/baz_test.ts +test test 2 ... ok ([WILDCARD]) +test test 3 ... ok ([WILDCARD]) +test test 6 ... ok ([WILDCARD]) +test test 9 ... ok ([WILDCARD]) +test test 8 ... ok ([WILDCARD]) +test test 7 ... ok ([WILDCARD]) +test test 5 ... ok ([WILDCARD]) +test test 4 ... ok ([WILDCARD]) +test test 1 ... ok ([WILDCARD]) +test test 0 ... ok ([WILDCARD]) +running 10 tests from [WILDCARD]/test/shuffle/bar_test.ts +test test 2 ... ok ([WILDCARD]) +test test 3 ... ok ([WILDCARD]) +test test 6 ... ok ([WILDCARD]) +test test 9 ... ok ([WILDCARD]) +test test 8 ... ok ([WILDCARD]) +test test 7 ... ok ([WILDCARD]) +test test 5 ... ok ([WILDCARD]) +test test 4 ... ok ([WILDCARD]) +test test 1 ... ok ([WILDCARD]) +test test 0 ... ok ([WILDCARD]) + +test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD]) + diff --git a/cli/tests/test/shuffle/bar_test.ts b/cli/tests/test/shuffle/bar_test.ts new file mode 100644 index 0000000000..ca118dc0de --- /dev/null +++ b/cli/tests/test/shuffle/bar_test.ts @@ -0,0 +1,3 @@ +for (let i = 0; i < 10; i++) { + Deno.test(`test ${i}`, () => {}); +} diff --git a/cli/tests/test/shuffle/baz_test.ts b/cli/tests/test/shuffle/baz_test.ts new file mode 100644 index 0000000000..ca118dc0de --- /dev/null +++ b/cli/tests/test/shuffle/baz_test.ts @@ -0,0 +1,3 @@ +for (let i = 0; i < 10; i++) { + Deno.test(`test ${i}`, () => {}); +} diff --git a/cli/tests/test/shuffle/foo_test.ts b/cli/tests/test/shuffle/foo_test.ts new file mode 100644 index 0000000000..ca118dc0de --- /dev/null +++ b/cli/tests/test/shuffle/foo_test.ts @@ -0,0 +1,3 @@ +for (let i = 0; i < 10; i++) { + Deno.test(`test ${i}`, () => {}); +} diff --git a/cli/tools/test_runner.rs b/cli/tools/test_runner.rs index 96b101fa33..d1a0c6d656 100644 --- a/cli/tools/test_runner.rs +++ b/cli/tools/test_runner.rs @@ -21,6 +21,9 @@ use deno_core::serde_json::json; use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_runtime::permissions::Permissions; +use rand::rngs::SmallRng; +use rand::seq::SliceRandom; +use rand::SeedableRng; use regex::Regex; use serde::Deserialize; use std::path::Path; @@ -343,8 +346,19 @@ pub async fn run_tests( quiet: bool, allow_none: bool, filter: Option, + shuffle: Option, concurrent_jobs: usize, ) -> Result { + let test_modules = if let Some(seed) = shuffle { + let mut rng = SmallRng::seed_from_u64(seed); + let mut test_modules = test_modules.clone(); + test_modules.sort(); + test_modules.shuffle(&mut rng); + test_modules + } else { + test_modules + }; + if !doc_modules.is_empty() { let mut test_programs = Vec::new(); @@ -450,6 +464,7 @@ pub async fn run_tests( let test_options = json!({ "disableLog": quiet, "filter": filter, + "shuffle": shuffle, }); let test_module = deno_core::resolve_path("$deno$test.js")?; diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js index 0106f895aa..4ee863ee71 100644 --- a/runtime/js/40_testing.js +++ b/runtime/js/40_testing.js @@ -217,6 +217,7 @@ finishing test case.`; async function runTests({ disableLog = false, filter = null, + shuffle = null, } = {}) { const originalConsole = globalThis.console; if (disableLog) { @@ -234,6 +235,24 @@ finishing test case.`; only: only.length > 0, }); + if (shuffle !== null) { + // http://en.wikipedia.org/wiki/Linear_congruential_generator + const nextInt = (function (state) { + const m = 0x80000000; + const a = 1103515245; + const c = 12345; + + return function (max) { + return state = ((a * state + c) % m) % max; + }; + }(shuffle)); + + for (let i = pending.length - 1; i > 0; i--) { + const j = nextInt(i); + [pending[i], pending[j]] = [pending[j], pending[i]]; + } + } + for (const test of pending) { const { name,