// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use crate::cache; use crate::cache::CacherLoader; use crate::colors; use crate::compat; use crate::create_main_worker; use crate::emit; use crate::file_watcher; use crate::file_watcher::ResolutionResult; use crate::flags::BenchFlags; use crate::flags::Flags; use crate::flags::TypeCheckMode; use crate::fs_util::collect_specifiers; use crate::fs_util::is_supported_bench_path; use crate::graph_util::contains_specifier; use crate::graph_util::graph_valid; use crate::located_script_name; use crate::lockfile; use crate::ops; use crate::proc_state::ProcState; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::future; use deno_core::futures::stream; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; use deno_core::serde_json::json; use deno_core::ModuleSpecifier; use deno_graph::ModuleKind; use deno_runtime::permissions::Permissions; use deno_runtime::tokio_util::run_basic; use log::Level; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; use std::path::PathBuf; use std::sync::Arc; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedSender; #[derive(Debug, Clone, Deserialize)] struct BenchSpecifierOptions { compat_mode: bool, filter: Option, } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub enum BenchOutput { Console(String), } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BenchPlan { pub total: usize, pub origin: String, pub used_only: bool, pub names: Vec, } #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] pub enum BenchEvent { Plan(BenchPlan), Output(BenchOutput), Wait(BenchMetadata), Result(String, BenchResult), } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub enum BenchResult { Ok(BenchMeasurement), Failed(BenchFailure), } #[derive(Debug, Clone, Serialize)] pub struct BenchReport { pub total: usize, pub failed: usize, pub failures: Vec, pub measurements: Vec, } #[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)] pub struct BenchMetadata { pub name: String, pub origin: String, pub baseline: bool, pub group: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BenchMeasurement { pub name: String, pub baseline: bool, pub stats: BenchStats, pub group: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BenchFailure { pub name: String, pub error: String, pub baseline: bool, pub group: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BenchStats { pub n: u64, pub min: f64, pub max: f64, pub avg: f64, pub p75: f64, pub p99: f64, pub p995: f64, pub p999: f64, } impl BenchReport { pub fn new() -> Self { Self { total: 0, failed: 0, failures: Vec::new(), measurements: Vec::new(), } } } fn create_reporter(show_output: bool) -> Box { Box::new(ConsoleReporter::new(show_output)) } pub trait BenchReporter { fn report_group_summary(&mut self); fn report_plan(&mut self, plan: &BenchPlan); fn report_end(&mut self, report: &BenchReport); fn report_wait(&mut self, wait: &BenchMetadata); fn report_output(&mut self, output: &BenchOutput); fn report_result(&mut self, result: &BenchResult); } struct ConsoleReporter { name: String, show_output: bool, has_ungrouped: bool, group: Option, baseline: Option, group_measurements: Vec, options: Option, } impl ConsoleReporter { fn new(show_output: bool) -> Self { Self { show_output, group: None, options: None, baseline: None, name: String::new(), has_ungrouped: false, group_measurements: Vec::new(), } } } impl BenchReporter for ConsoleReporter { #[cold] fn report_plan(&mut self, plan: &BenchPlan) { use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; static FIRST_PLAN: AtomicBool = AtomicBool::new(true); self.options = Some(mitata::reporter::Options::new( &plan.names.iter().map(|x| x.as_str()).collect::>(), )); let options = self.options.as_mut().unwrap(); options.percentiles = true; options.colors = colors::use_color(); if FIRST_PLAN .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst) .is_ok() { println!("{}", colors::gray(format!("cpu: {}", mitata::cpu::name()))); println!( "{}\n", colors::gray(format!( "runtime: deno {} ({})", crate::version::deno(), env!("TARGET") )) ); } else { println!(); } println!( "{}\n{}\n{}", colors::gray(&plan.origin), mitata::reporter::header(options), mitata::reporter::br(options) ); } fn report_wait(&mut self, wait: &BenchMetadata) { self.name = wait.name.clone(); match &wait.group { None => { self.has_ungrouped = true; } Some(group) => { if self.group.is_none() && self.has_ungrouped && self.group_measurements.is_empty() { println!(); } if None == self.group || group != self.group.as_ref().unwrap() { self.report_group_summary(); } if (self.group.is_none() && self.has_ungrouped) || (self.group.is_some() && self.group_measurements.is_empty()) { println!(); } self.group = Some(group.clone()); } } } fn report_output(&mut self, output: &BenchOutput) { if self.show_output { match output { BenchOutput::Console(line) => { print!("{} {}", colors::gray(format!("{}:", self.name)), line) } } } } fn report_result(&mut self, result: &BenchResult) { let options = self.options.as_ref().unwrap(); match result { BenchResult::Ok(bench) => { let mut bench = bench.to_owned(); if bench.baseline && self.baseline.is_none() { self.baseline = Some(bench.clone()); } else { bench.baseline = false; } self.group_measurements.push(bench.clone()); println!( "{}", mitata::reporter::benchmark( &bench.name, &mitata::reporter::BenchmarkStats { avg: bench.stats.avg, min: bench.stats.min, max: bench.stats.max, p75: bench.stats.p75, p99: bench.stats.p99, p995: bench.stats.p995, }, options ) ); } BenchResult::Failed(failure) => { println!( "{}", mitata::reporter::benchmark_error( &failure.name, &mitata::reporter::Error { stack: None, message: failure.error.clone(), }, options ) ) } }; } fn report_group_summary(&mut self) { let options = match self.options.as_ref() { None => return, Some(options) => options, }; if 2 <= self.group_measurements.len() && (self.group.is_some() || (self.group.is_none() && self.baseline.is_some())) { println!( "\n{}", mitata::reporter::summary( &self .group_measurements .iter() .map(|b| mitata::reporter::GroupBenchmark { name: b.name.clone(), baseline: b.baseline, group: b.group.as_deref().unwrap_or("").to_owned(), stats: mitata::reporter::BenchmarkStats { avg: b.stats.avg, min: b.stats.min, max: b.stats.max, p75: b.stats.p75, p99: b.stats.p99, p995: b.stats.p995, }, }) .collect::>(), options ) ); } self.baseline = None; self.group_measurements.clear(); } fn report_end(&mut self, _: &BenchReport) { self.report_group_summary(); } } /// Type check a collection of module and document specifiers. async fn check_specifiers( ps: &ProcState, permissions: Permissions, specifiers: Vec, lib: emit::TypeLib, ) -> Result<(), AnyError> { ps.prepare_module_load( specifiers, false, lib, Permissions::allow_all(), permissions, true, ) .await?; Ok(()) } /// Run a single specifier as an executable bench module. async fn bench_specifier( ps: ProcState, permissions: Permissions, specifier: ModuleSpecifier, channel: UnboundedSender, options: BenchSpecifierOptions, ) -> Result<(), AnyError> { let mut worker = create_main_worker( &ps, specifier.clone(), permissions, vec![ops::bench::init(channel.clone(), ps.flags.unstable)], ); if options.compat_mode { worker.execute_side_module(&compat::GLOBAL_URL).await?; worker.execute_side_module(&compat::MODULE_URL).await?; let use_esm_loader = compat::check_if_should_use_esm_loader(&specifier)?; if use_esm_loader { worker.execute_side_module(&specifier).await?; } else { compat::load_cjs_module( &mut worker.js_runtime, &specifier.to_file_path().unwrap().display().to_string(), false, )?; worker.run_event_loop(false).await?; } } else { // We execute the module module as a side module so that import.meta.main is not set. worker.execute_side_module(&specifier).await?; } worker.dispatch_load_event(&located_script_name!())?; let bench_result = worker.js_runtime.execute_script( &located_script_name!(), &format!( r#"Deno[Deno.internal].runBenchmarks({})"#, json!({ "filter": options.filter, }), ), )?; worker.js_runtime.resolve_value(bench_result).await?; worker.dispatch_unload_event(&located_script_name!())?; Ok(()) } /// Test a collection of specifiers with test modes concurrently. async fn bench_specifiers( ps: ProcState, permissions: Permissions, specifiers: Vec, options: BenchSpecifierOptions, ) -> Result<(), AnyError> { let log_level = ps.flags.log_level; let (sender, mut receiver) = unbounded_channel::(); let join_handles = specifiers.iter().map(move |specifier| { let ps = ps.clone(); let permissions = permissions.clone(); let specifier = specifier.clone(); let sender = sender.clone(); let options = options.clone(); tokio::task::spawn_blocking(move || { let future = bench_specifier(ps, permissions, specifier, sender, options); run_basic(future) }) }); let join_stream = stream::iter(join_handles) .buffer_unordered(1) .collect::, tokio::task::JoinError>>>(); let handler = { tokio::task::spawn(async move { let mut used_only = false; let mut report = BenchReport::new(); let mut reporter = create_reporter(log_level != Some(Level::Error)); while let Some(event) = receiver.recv().await { match event { BenchEvent::Plan(plan) => { report.total += plan.total; if plan.used_only { used_only = true; } reporter.report_plan(&plan); } BenchEvent::Wait(metadata) => { reporter.report_wait(&metadata); } BenchEvent::Output(output) => { reporter.report_output(&output); } BenchEvent::Result(_origin, result) => { match &result { BenchResult::Ok(bench) => { report.measurements.push(bench.clone()); } BenchResult::Failed(failure) => { report.failed += 1; report.failures.push(failure.clone()); } }; reporter.report_result(&result); } } } reporter.report_end(&report); if used_only { return Err(generic_error( "Bench failed because the \"only\" option was used", )); } if report.failed > 0 { return Err(generic_error("Bench failed")); } Ok(()) }) }; let (join_results, result) = future::join(join_stream, handler).await; // propagate any errors for join_result in join_results { join_result??; } result??; Ok(()) } pub async fn run_benchmarks( flags: Flags, bench_flags: BenchFlags, ) -> Result<(), AnyError> { let ps = ProcState::build(Arc::new(flags)).await?; let permissions = Permissions::from_options(&ps.flags.permissions_options()); let specifiers = collect_specifiers( bench_flags.include.unwrap_or_else(|| vec![".".to_string()]), &bench_flags.ignore.clone(), is_supported_bench_path, )?; if specifiers.is_empty() { return Err(generic_error("No bench modules found")); } let lib = if ps.flags.unstable { emit::TypeLib::UnstableDenoWindow } else { emit::TypeLib::DenoWindow }; check_specifiers(&ps, permissions.clone(), specifiers.clone(), lib).await?; let compat = ps.flags.compat; bench_specifiers( ps, permissions, specifiers, BenchSpecifierOptions { compat_mode: compat, filter: bench_flags.filter, }, ) .await?; Ok(()) } // TODO(bartlomieju): heavy duplication of code with `cli/tools/test.rs` pub async fn run_benchmarks_with_watch( flags: Flags, bench_flags: BenchFlags, ) -> Result<(), AnyError> { let flags = Arc::new(flags); let ps = ProcState::build(flags.clone()).await?; let permissions = Permissions::from_options(&flags.permissions_options()); let lib = if flags.unstable { emit::TypeLib::UnstableDenoWindow } else { emit::TypeLib::DenoWindow }; let include = bench_flags.include.unwrap_or_else(|| vec![".".to_string()]); let ignore = bench_flags.ignore.clone(); let paths_to_watch: Vec<_> = include.iter().map(PathBuf::from).collect(); let no_check = ps.flags.type_check_mode == TypeCheckMode::None; let resolver = |changed: Option>| { let mut cache = cache::FetchCacher::new( ps.dir.gen_cache.clone(), ps.file_fetcher.clone(), Permissions::allow_all(), Permissions::allow_all(), ); let paths_to_watch = paths_to_watch.clone(); let paths_to_watch_clone = paths_to_watch.clone(); let maybe_import_map_resolver = ps.maybe_import_map.clone().map(ImportMapResolver::new); let maybe_jsx_resolver = ps.maybe_config_file.as_ref().and_then(|cf| { cf.to_maybe_jsx_import_source_module() .map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone())) }); let maybe_locker = lockfile::as_maybe_locker(ps.lockfile.clone()); let maybe_imports = ps .maybe_config_file .as_ref() .map(|cf| cf.to_maybe_imports()); let files_changed = changed.is_some(); let include = include.clone(); let ignore = ignore.clone(); let check_js = ps .maybe_config_file .as_ref() .map(|cf| cf.get_check_js()) .unwrap_or(false); async move { let bench_modules = collect_specifiers(include.clone(), &ignore, is_supported_bench_path)?; let mut paths_to_watch = paths_to_watch_clone; let mut modules_to_reload = if files_changed { Vec::new() } else { bench_modules .iter() .map(|url| (url.clone(), ModuleKind::Esm)) .collect() }; let maybe_imports = if let Some(result) = maybe_imports { result? } else { None }; let maybe_resolver = if maybe_jsx_resolver.is_some() { maybe_jsx_resolver.as_ref().map(|jr| jr.as_resolver()) } else { maybe_import_map_resolver .as_ref() .map(|im| im.as_resolver()) }; let graph = deno_graph::create_graph( bench_modules .iter() .map(|s| (s.clone(), ModuleKind::Esm)) .collect(), false, maybe_imports, cache.as_mut_loader(), maybe_resolver, maybe_locker, None, None, ) .await; graph_valid(&graph, !no_check, check_js)?; // TODO(@kitsonk) - This should be totally derivable from the graph. for specifier in bench_modules { fn get_dependencies<'a>( graph: &'a deno_graph::ModuleGraph, maybe_module: Option<&'a deno_graph::Module>, // This needs to be accessible to skip getting dependencies if they're already there, // otherwise this will cause a stack overflow with circular dependencies output: &mut HashSet<&'a ModuleSpecifier>, no_check: bool, ) { if let Some(module) = maybe_module { for dep in module.dependencies.values() { if let Some(specifier) = &dep.get_code() { if !output.contains(specifier) { output.insert(specifier); get_dependencies( graph, graph.get(specifier), output, no_check, ); } } if !no_check { if let Some(specifier) = &dep.get_type() { if !output.contains(specifier) { output.insert(specifier); get_dependencies( graph, graph.get(specifier), output, no_check, ); } } } } } } // This bench module and all it's dependencies let mut modules = HashSet::new(); modules.insert(&specifier); get_dependencies(&graph, graph.get(&specifier), &mut modules, no_check); paths_to_watch.extend( modules .iter() .filter_map(|specifier| specifier.to_file_path().ok()), ); if let Some(changed) = &changed { for path in changed.iter().filter_map(|path| { deno_core::resolve_url_or_path(&path.to_string_lossy()).ok() }) { if modules.contains(&&path) { modules_to_reload.push((specifier, ModuleKind::Esm)); break; } } } } Ok((paths_to_watch, modules_to_reload)) } .map(move |result| { if files_changed && matches!(result, Ok((_, ref modules)) if modules.is_empty()) { ResolutionResult::Ignore } else { match result { Ok((paths_to_watch, modules_to_reload)) => { ResolutionResult::Restart { paths_to_watch, result: Ok(modules_to_reload), } } Err(e) => ResolutionResult::Restart { paths_to_watch, result: Err(e), }, } } }) }; let operation = |modules_to_reload: Vec<(ModuleSpecifier, ModuleKind)>| { let flags = flags.clone(); let filter = bench_flags.filter.clone(); let include = include.clone(); let ignore = ignore.clone(); let lib = lib.clone(); let permissions = permissions.clone(); let ps = ps.clone(); async move { let specifiers = collect_specifiers(include.clone(), &ignore, is_supported_bench_path)? .iter() .filter(|specifier| contains_specifier(&modules_to_reload, specifier)) .cloned() .collect::>(); check_specifiers(&ps, permissions.clone(), specifiers.clone(), lib) .await?; bench_specifiers( ps, permissions.clone(), specifiers, BenchSpecifierOptions { compat_mode: flags.compat, filter: filter.clone(), }, ) .await?; Ok(()) } }; file_watcher::watch_func( resolver, operation, file_watcher::PrintConfig { job_name: "Bench".to_string(), clear_screen: !flags.no_clear_screen, }, ) .await?; Ok(()) }