1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-25 00:29:09 -05:00

feat(test): add --shuffle flag to randomize test ordering (#11163)

This commit is contained in:
Casper Beyer 2021-07-06 09:20:33 +08:00 committed by GitHub
parent bdeb4f430b
commit e8258e0210
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 136 additions and 8 deletions

15
Cargo.lock generated
View file

@ -568,6 +568,7 @@ dependencies = [
"os_pipe", "os_pipe",
"percent-encoding", "percent-encoding",
"pin-project", "pin-project",
"rand 0.8.4",
"regex", "regex",
"ring", "ring",
"rustyline", "rustyline",
@ -645,7 +646,7 @@ version = "0.24.1"
dependencies = [ dependencies = [
"deno_core", "deno_core",
"deno_web", "deno_web",
"rand 0.8.3", "rand 0.8.4",
"ring", "ring",
"tokio", "tokio",
"uuid", "uuid",
@ -2529,9 +2530,9 @@ dependencies = [
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.3" version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [ dependencies = [
"libc", "libc",
"rand_chacha 0.3.0", "rand_chacha 0.3.0",
@ -3553,7 +3554,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"libc", "libc",
"rand 0.8.3", "rand 0.8.4",
"redox_syscall", "redox_syscall",
"remove_dir_all", "remove_dir_all",
"winapi 0.3.9", "winapi 0.3.9",
@ -3831,7 +3832,7 @@ dependencies = [
"lazy_static", "lazy_static",
"log", "log",
"radix_trie", "radix_trie",
"rand 0.8.3", "rand 0.8.4",
"thiserror", "thiserror",
"tokio", "tokio",
"trust-dns-proto", "trust-dns-proto",
@ -3854,7 +3855,7 @@ dependencies = [
"ipnet", "ipnet",
"lazy_static", "lazy_static",
"log", "log",
"rand 0.8.3", "rand 0.8.4",
"serde", "serde",
"smallvec", "smallvec",
"thiserror", "thiserror",
@ -3926,7 +3927,7 @@ dependencies = [
"httparse", "httparse",
"input_buffer", "input_buffer",
"log", "log",
"rand 0.8.3", "rand 0.8.4",
"rustls", "rustls",
"sha-1", "sha-1",
"thiserror", "thiserror",

View file

@ -71,6 +71,7 @@ num_cpus = "1.13.0"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
pin-project = "1.0.6" pin-project = "1.0.6"
regex = "1.4.3" regex = "1.4.3"
rand = { version = "0.8.3", features = [ "small_rng" ] }
ring = "0.16.20" ring = "0.16.20"
rustyline = { version = "8.0.0", default-features = false } rustyline = { version = "8.0.0", default-features = false }
rustyline-derive = "0.4.0" rustyline-derive = "0.4.0"

View file

@ -103,6 +103,7 @@ pub enum DenoSubcommand {
allow_none: bool, allow_none: bool,
include: Option<Vec<String>>, include: Option<Vec<String>>,
filter: Option<String>, filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize, concurrent_jobs: usize,
}, },
Types, Types,
@ -1016,6 +1017,20 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
.takes_value(true) .takes_value(true)
.help("Run tests with this string or pattern in the test name"), .help("Run tests with this string or pattern in the test name"),
) )
.arg(
Arg::with_name("shuffle")
.long("shuffle")
.value_name("NUMBER")
.help("(UNSTABLE): Shuffle the order in which the tests are run")
.min_values(0)
.max_values(1)
.require_equals(true)
.takes_value(true)
.validator(|val: String| match val.parse::<u64>() {
Ok(_) => Ok(()),
Err(_) => Err("Shuffle seed should be a number".to_string()),
}),
)
.arg( .arg(
Arg::with_name("coverage") Arg::with_name("coverage")
.long("coverage") .long("coverage")
@ -1686,7 +1701,17 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
let quiet = matches.is_present("quiet"); let quiet = matches.is_present("quiet");
let filter = matches.value_of("filter").map(String::from); let filter = matches.value_of("filter").map(String::from);
flags.watch = matches.is_present("watch"); let shuffle = if matches.is_present("shuffle") {
let value = if let Some(value) = matches.value_of("shuffle") {
value.parse::<u64>().unwrap()
} else {
rand::random::<u64>()
};
Some(value)
} else {
None
};
if matches.is_present("script_arg") { if matches.is_present("script_arg") {
let script_arg: Vec<String> = matches let script_arg: Vec<String> = matches
@ -1730,6 +1755,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
quiet, quiet,
include, include,
filter, filter,
shuffle,
allow_none, allow_none,
concurrent_jobs, concurrent_jobs,
}; };
@ -3366,6 +3392,7 @@ mod tests {
allow_none: true, allow_none: true,
quiet: false, quiet: false,
include: Some(svec!["dir1/", "dir2/"]), include: Some(svec!["dir1/", "dir2/"]),
shuffle: None,
concurrent_jobs: 1, concurrent_jobs: 1,
}, },
unstable: true, unstable: true,

View file

@ -984,6 +984,7 @@ async fn test_command(
quiet: bool, quiet: bool,
allow_none: bool, allow_none: bool,
filter: Option<String>, filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize, concurrent_jobs: usize,
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
if let Some(ref coverage_dir) = flags.coverage_dir { if let Some(ref coverage_dir) = flags.coverage_dir {
@ -1172,6 +1173,7 @@ async fn test_command(
quiet, quiet,
true, true,
filter.clone(), filter.clone(),
shuffle,
concurrent_jobs, concurrent_jobs,
) )
.map(|res| res.map(|_| ())) .map(|res| res.map(|_| ()))
@ -1207,6 +1209,7 @@ async fn test_command(
quiet, quiet,
allow_none, allow_none,
filter, filter,
shuffle,
concurrent_jobs, concurrent_jobs,
) )
.await?; .await?;
@ -1314,6 +1317,7 @@ fn get_subcommand(
include, include,
allow_none, allow_none,
filter, filter,
shuffle,
concurrent_jobs, concurrent_jobs,
} => test_command( } => test_command(
flags, flags,
@ -1324,6 +1328,7 @@ fn get_subcommand(
quiet, quiet,
allow_none, allow_none,
filter, filter,
shuffle,
concurrent_jobs, concurrent_jobs,
) )
.boxed_local(), .boxed_local(),

View file

@ -114,3 +114,15 @@ itest!(unhandled_rejection {
exit_code: 1, exit_code: 1,
output: "test/unhandled_rejection.out", output: "test/unhandled_rejection.out",
}); });
itest!(shuffle {
args: "test --shuffle test/shuffle",
exit_code: 0,
output_str: Some("[WILDCARD]"),
});
itest!(shuffle_with_seed {
args: "test --shuffle=42 test/shuffle",
exit_code: 0,
output: "test/shuffle.out",
});

View file

@ -0,0 +1,39 @@
Check [WILDCARD]/test/shuffle/foo_test.ts
Check [WILDCARD]/test/shuffle/baz_test.ts
Check [WILDCARD]/test/shuffle/bar_test.ts
running 10 tests from [WILDCARD]/test/shuffle/foo_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
running 10 tests from [WILDCARD]/test/shuffle/baz_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
running 10 tests from [WILDCARD]/test/shuffle/bar_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -21,6 +21,9 @@ use deno_core::serde_json::json;
use deno_core::url::Url; use deno_core::url::Url;
use deno_core::ModuleSpecifier; use deno_core::ModuleSpecifier;
use deno_runtime::permissions::Permissions; use deno_runtime::permissions::Permissions;
use rand::rngs::SmallRng;
use rand::seq::SliceRandom;
use rand::SeedableRng;
use regex::Regex; use regex::Regex;
use serde::Deserialize; use serde::Deserialize;
use std::path::Path; use std::path::Path;
@ -343,8 +346,19 @@ pub async fn run_tests(
quiet: bool, quiet: bool,
allow_none: bool, allow_none: bool,
filter: Option<String>, filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize, concurrent_jobs: usize,
) -> Result<bool, AnyError> { ) -> Result<bool, AnyError> {
let test_modules = if let Some(seed) = shuffle {
let mut rng = SmallRng::seed_from_u64(seed);
let mut test_modules = test_modules.clone();
test_modules.sort();
test_modules.shuffle(&mut rng);
test_modules
} else {
test_modules
};
if !doc_modules.is_empty() { if !doc_modules.is_empty() {
let mut test_programs = Vec::new(); let mut test_programs = Vec::new();
@ -450,6 +464,7 @@ pub async fn run_tests(
let test_options = json!({ let test_options = json!({
"disableLog": quiet, "disableLog": quiet,
"filter": filter, "filter": filter,
"shuffle": shuffle,
}); });
let test_module = deno_core::resolve_path("$deno$test.js")?; let test_module = deno_core::resolve_path("$deno$test.js")?;

View file

@ -217,6 +217,7 @@ finishing test case.`;
async function runTests({ async function runTests({
disableLog = false, disableLog = false,
filter = null, filter = null,
shuffle = null,
} = {}) { } = {}) {
const originalConsole = globalThis.console; const originalConsole = globalThis.console;
if (disableLog) { if (disableLog) {
@ -234,6 +235,24 @@ finishing test case.`;
only: only.length > 0, only: only.length > 0,
}); });
if (shuffle !== null) {
// http://en.wikipedia.org/wiki/Linear_congruential_generator
const nextInt = (function (state) {
const m = 0x80000000;
const a = 1103515245;
const c = 12345;
return function (max) {
return state = ((a * state + c) % m) % max;
};
}(shuffle));
for (let i = pending.length - 1; i > 0; i--) {
const j = nextInt(i);
[pending[i], pending[j]] = [pending[j], pending[i]];
}
}
for (const test of pending) { for (const test of pending) {
const { const {
name, name,