1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-25 00:29:09 -05:00

feat(test): add --shuffle flag to randomize test ordering (#11163)

This commit is contained in:
Casper Beyer 2021-07-06 09:20:33 +08:00 committed by GitHub
parent bdeb4f430b
commit e8258e0210
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 136 additions and 8 deletions

15
Cargo.lock generated
View file

@ -568,6 +568,7 @@ dependencies = [
"os_pipe",
"percent-encoding",
"pin-project",
"rand 0.8.4",
"regex",
"ring",
"rustyline",
@ -645,7 +646,7 @@ version = "0.24.1"
dependencies = [
"deno_core",
"deno_web",
"rand 0.8.3",
"rand 0.8.4",
"ring",
"tokio",
"uuid",
@ -2529,9 +2530,9 @@ dependencies = [
[[package]]
name = "rand"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
dependencies = [
"libc",
"rand_chacha 0.3.0",
@ -3553,7 +3554,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [
"cfg-if 1.0.0",
"libc",
"rand 0.8.3",
"rand 0.8.4",
"redox_syscall",
"remove_dir_all",
"winapi 0.3.9",
@ -3831,7 +3832,7 @@ dependencies = [
"lazy_static",
"log",
"radix_trie",
"rand 0.8.3",
"rand 0.8.4",
"thiserror",
"tokio",
"trust-dns-proto",
@ -3854,7 +3855,7 @@ dependencies = [
"ipnet",
"lazy_static",
"log",
"rand 0.8.3",
"rand 0.8.4",
"serde",
"smallvec",
"thiserror",
@ -3926,7 +3927,7 @@ dependencies = [
"httparse",
"input_buffer",
"log",
"rand 0.8.3",
"rand 0.8.4",
"rustls",
"sha-1",
"thiserror",

View file

@ -71,6 +71,7 @@ num_cpus = "1.13.0"
percent-encoding = "2.1.0"
pin-project = "1.0.6"
regex = "1.4.3"
rand = { version = "0.8.3", features = [ "small_rng" ] }
ring = "0.16.20"
rustyline = { version = "8.0.0", default-features = false }
rustyline-derive = "0.4.0"

View file

@ -103,6 +103,7 @@ pub enum DenoSubcommand {
allow_none: bool,
include: Option<Vec<String>>,
filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize,
},
Types,
@ -1016,6 +1017,20 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
.takes_value(true)
.help("Run tests with this string or pattern in the test name"),
)
.arg(
Arg::with_name("shuffle")
.long("shuffle")
.value_name("NUMBER")
.help("(UNSTABLE): Shuffle the order in which the tests are run")
.min_values(0)
.max_values(1)
.require_equals(true)
.takes_value(true)
.validator(|val: String| match val.parse::<u64>() {
Ok(_) => Ok(()),
Err(_) => Err("Shuffle seed should be a number".to_string()),
}),
)
.arg(
Arg::with_name("coverage")
.long("coverage")
@ -1686,7 +1701,17 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
let quiet = matches.is_present("quiet");
let filter = matches.value_of("filter").map(String::from);
flags.watch = matches.is_present("watch");
let shuffle = if matches.is_present("shuffle") {
let value = if let Some(value) = matches.value_of("shuffle") {
value.parse::<u64>().unwrap()
} else {
rand::random::<u64>()
};
Some(value)
} else {
None
};
if matches.is_present("script_arg") {
let script_arg: Vec<String> = matches
@ -1730,6 +1755,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
quiet,
include,
filter,
shuffle,
allow_none,
concurrent_jobs,
};
@ -3366,6 +3392,7 @@ mod tests {
allow_none: true,
quiet: false,
include: Some(svec!["dir1/", "dir2/"]),
shuffle: None,
concurrent_jobs: 1,
},
unstable: true,

View file

@ -984,6 +984,7 @@ async fn test_command(
quiet: bool,
allow_none: bool,
filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize,
) -> Result<(), AnyError> {
if let Some(ref coverage_dir) = flags.coverage_dir {
@ -1172,6 +1173,7 @@ async fn test_command(
quiet,
true,
filter.clone(),
shuffle,
concurrent_jobs,
)
.map(|res| res.map(|_| ()))
@ -1207,6 +1209,7 @@ async fn test_command(
quiet,
allow_none,
filter,
shuffle,
concurrent_jobs,
)
.await?;
@ -1314,6 +1317,7 @@ fn get_subcommand(
include,
allow_none,
filter,
shuffle,
concurrent_jobs,
} => test_command(
flags,
@ -1324,6 +1328,7 @@ fn get_subcommand(
quiet,
allow_none,
filter,
shuffle,
concurrent_jobs,
)
.boxed_local(),

View file

@ -114,3 +114,15 @@ itest!(unhandled_rejection {
exit_code: 1,
output: "test/unhandled_rejection.out",
});
itest!(shuffle {
args: "test --shuffle test/shuffle",
exit_code: 0,
output_str: Some("[WILDCARD]"),
});
itest!(shuffle_with_seed {
args: "test --shuffle=42 test/shuffle",
exit_code: 0,
output: "test/shuffle.out",
});

View file

@ -0,0 +1,39 @@
Check [WILDCARD]/test/shuffle/foo_test.ts
Check [WILDCARD]/test/shuffle/baz_test.ts
Check [WILDCARD]/test/shuffle/bar_test.ts
running 10 tests from [WILDCARD]/test/shuffle/foo_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
running 10 tests from [WILDCARD]/test/shuffle/baz_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
running 10 tests from [WILDCARD]/test/shuffle/bar_test.ts
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 0 ... ok ([WILDCARD])
test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -0,0 +1,3 @@
for (let i = 0; i < 10; i++) {
Deno.test(`test ${i}`, () => {});
}

View file

@ -21,6 +21,9 @@ use deno_core::serde_json::json;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::Permissions;
use rand::rngs::SmallRng;
use rand::seq::SliceRandom;
use rand::SeedableRng;
use regex::Regex;
use serde::Deserialize;
use std::path::Path;
@ -343,8 +346,19 @@ pub async fn run_tests(
quiet: bool,
allow_none: bool,
filter: Option<String>,
shuffle: Option<u64>,
concurrent_jobs: usize,
) -> Result<bool, AnyError> {
let test_modules = if let Some(seed) = shuffle {
let mut rng = SmallRng::seed_from_u64(seed);
let mut test_modules = test_modules.clone();
test_modules.sort();
test_modules.shuffle(&mut rng);
test_modules
} else {
test_modules
};
if !doc_modules.is_empty() {
let mut test_programs = Vec::new();
@ -450,6 +464,7 @@ pub async fn run_tests(
let test_options = json!({
"disableLog": quiet,
"filter": filter,
"shuffle": shuffle,
});
let test_module = deno_core::resolve_path("$deno$test.js")?;

View file

@ -217,6 +217,7 @@ finishing test case.`;
async function runTests({
disableLog = false,
filter = null,
shuffle = null,
} = {}) {
const originalConsole = globalThis.console;
if (disableLog) {
@ -234,6 +235,24 @@ finishing test case.`;
only: only.length > 0,
});
if (shuffle !== null) {
// http://en.wikipedia.org/wiki/Linear_congruential_generator
const nextInt = (function (state) {
const m = 0x80000000;
const a = 1103515245;
const c = 12345;
return function (max) {
return state = ((a * state + c) % m) % max;
};
}(shuffle));
for (let i = pending.length - 1; i > 0; i--) {
const j = nextInt(i);
[pending[i], pending[j]] = [pending[j], pending[i]];
}
}
for (const test of pending) {
const {
name,