From f92bd986de7f083ac164c0e3a2b04d3504c29741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Fri, 9 Sep 2022 21:57:39 +0200 Subject: [PATCH] feat: download progress bar (#15814) --- Cargo.lock | 48 +++++++++++++++ cli/Cargo.toml | 1 + cli/file_fetcher.rs | 30 ++++++++-- cli/lsp/registries.rs | 1 + cli/main.rs | 1 + cli/npm/cache.rs | 18 +++--- cli/npm/mod.rs | 9 ++- cli/npm/registry.rs | 21 ++++--- cli/proc_state.rs | 8 +++ cli/progress_bar.rs | 132 ++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 244 insertions(+), 25 deletions(-) create mode 100644 cli/progress_bar.rs diff --git a/Cargo.lock b/Cargo.lock index a71b6c0878..3f958dc041 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -526,6 +526,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "console" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89eab4d20ce20cea182308bca13088fecea9c05f6776cf287205d41a0ed3c847" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "terminal_size", + "unicode-width", + "winapi 0.3.9", +] + [[package]] name = "const-oid" version = "0.9.0" @@ -825,6 +839,7 @@ dependencies = [ "http", "import_map", "indexmap", + "indicatif", "jsonc-parser", "libc", "log 0.4.17", @@ -1535,6 +1550,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" version = "0.8.31" @@ -2289,6 +2310,17 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc42b206e70d86ec03285b123e65a5458c92027d1fb2ae3555878b8113b3ddf" +dependencies = [ + "console", + "number_prefix", + "unicode-width", +] + [[package]] name = "inotify" version = "0.9.6" @@ -2932,6 +2964,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "objc" version = "0.2.7" @@ -4635,6 +4673,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "test_ffi" version = "0.1.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 021195bb46..eaef87e000 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -75,6 +75,7 @@ flate2 = "=1.0.24" http = "=0.2.6" import_map = "=0.12.1" indexmap = "1.8.1" +indicatif = "=0.17.0" jsonc-parser = { version = "=0.21.0", features = ["serde"] } libc = "=0.2.126" log = { version = "=0.4.17", features = ["serde"] } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 0e01237d94..044707cff3 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -7,6 +7,7 @@ use crate::http_util::fetch_once; use crate::http_util::CacheSemantics; use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceResult; +use crate::progress_bar::ProgressBar; use crate::text_encoding; use crate::version::get_user_agent; @@ -318,6 +319,7 @@ pub struct FileFetcher { http_client: reqwest::Client, blob_store: BlobStore, download_log_level: log::Level, + progress_bar: Option, } impl FileFetcher { @@ -328,6 +330,7 @@ impl FileFetcher { root_cert_store: Option, blob_store: BlobStore, unsafely_ignore_certificate_errors: Option>, + progress_bar: Option, ) -> Result { Ok(Self { auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), @@ -345,6 +348,7 @@ impl FileFetcher { )?, blob_store, download_log_level: log::Level::Info, + progress_bar, }) } @@ -584,12 +588,17 @@ impl FileFetcher { .boxed(); } - log::log!( - self.download_log_level, - "{} {}", - colors::green("Download"), - specifier - ); + let mut _maybe_guard = None; + if let Some(pb) = self.progress_bar.as_ref() { + _maybe_guard = Some(pb.update(specifier.as_str())); + } else { + log::log!( + self.download_log_level, + "{} {}", + colors::green("Download"), + specifier + ); + } let maybe_etag = match self.http_cache.get(specifier) { Ok((_, headers, _)) => headers.get("etag").cloned(), @@ -770,6 +779,7 @@ mod tests { None, blob_store.clone(), None, + None, ) .unwrap(); (file_fetcher, temp_dir, blob_store) @@ -1208,6 +1218,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let result = file_fetcher @@ -1234,6 +1245,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let specifier = @@ -1261,6 +1273,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let result = file_fetcher_02 @@ -1404,6 +1417,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let specifier = @@ -1433,6 +1447,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let result = file_fetcher_02 @@ -1533,6 +1548,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let specifier = resolve_url("http://localhost:4545/002_hello.ts").unwrap(); @@ -1558,6 +1574,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let file_fetcher_02 = FileFetcher::new( @@ -1567,6 +1584,7 @@ mod tests { None, BlobStore::default(), None, + None, ) .unwrap(); let specifier = resolve_url("http://localhost:4545/002_hello.ts").unwrap(); diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index c9ed5da63d..3da435666f 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -455,6 +455,7 @@ impl ModuleRegistry { root_cert_store, BlobStore::default(), options.unsafely_ignore_certificate_errors, + None, )?; file_fetcher.set_download_log_level(super::logging::lsp_log_level()); diff --git a/cli/main.rs b/cli/main.rs index 98b5325b09..1c9c10281e 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -26,6 +26,7 @@ mod node; mod npm; mod ops; mod proc_state; +mod progress_bar; mod resolver; mod standalone; mod text_encoding; diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index e39957a6bf..a0082fb29a 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -11,12 +11,12 @@ use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::url::Url; -use deno_runtime::colors; use deno_runtime::deno_fetch::reqwest; use crate::deno_dir::DenoDir; use crate::file_fetcher::CacheSetting; use crate::fs_util; +use crate::progress_bar::ProgressBar; use super::semver::NpmVersion; use super::tarball::verify_and_extract_tarball; @@ -173,13 +173,19 @@ impl ReadonlyNpmCache { pub struct NpmCache { readonly: ReadonlyNpmCache, cache_setting: CacheSetting, + progress_bar: ProgressBar, } impl NpmCache { - pub fn from_deno_dir(dir: &DenoDir, cache_setting: CacheSetting) -> Self { + pub fn from_deno_dir( + dir: &DenoDir, + cache_setting: CacheSetting, + progress_bar: ProgressBar, + ) -> Self { Self { readonly: ReadonlyNpmCache::from_deno_dir(dir), cache_setting, + progress_bar, } } @@ -211,13 +217,7 @@ impl NpmCache { ); } - log::log!( - log::Level::Info, - "{} {}", - colors::green("Download"), - dist.tarball, - ); - + let _guard = self.progress_bar.update(&dist.tarball); let response = reqwest::get(&dist.tarball).await?; if response.status() == 404 { diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index d0a57c5bc0..2dd871cffe 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -31,6 +31,7 @@ use resolution::NpmResolution; use crate::deno_dir::DenoDir; use crate::file_fetcher::CacheSetting; +use crate::progress_bar::ProgressBar; use self::cache::ReadonlyNpmCache; use self::resolution::NpmResolutionSnapshot; @@ -87,13 +88,15 @@ impl GlobalNpmPackageResolver { cache_setting: CacheSetting, unstable: bool, no_npm: bool, + progress_bar: ProgressBar, ) -> Self { Self::from_cache( - NpmCache::from_deno_dir(dir, cache_setting.clone()), + NpmCache::from_deno_dir(dir, cache_setting.clone(), progress_bar.clone()), reload, cache_setting, unstable, no_npm, + progress_bar, ) } @@ -103,8 +106,10 @@ impl GlobalNpmPackageResolver { cache_setting: CacheSetting, unstable: bool, no_npm: bool, + progress_bar: ProgressBar, ) -> Self { - let api = NpmRegistryApi::new(cache.clone(), reload, cache_setting); + let api = + NpmRegistryApi::new(cache.clone(), reload, cache_setting, progress_bar); let registry_url = api.base_url().to_owned(); let resolution = Arc::new(NpmResolution::new(api)); diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 1fb4b2e0a6..f5b8b9fc80 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -21,6 +21,7 @@ use serde::Serialize; use crate::file_fetcher::CacheSetting; use crate::fs_util; use crate::http_cache::CACHE_PERM; +use crate::progress_bar::ProgressBar; use super::cache::NpmCache; use super::semver::NpmVersionReq; @@ -106,6 +107,7 @@ pub struct NpmRegistryApi { mem_cache: Arc>>>, reload: bool, cache_setting: CacheSetting, + progress_bar: ProgressBar, } impl NpmRegistryApi { @@ -132,8 +134,15 @@ impl NpmRegistryApi { cache: NpmCache, reload: bool, cache_setting: CacheSetting, + progress_bar: ProgressBar, ) -> Self { - Self::from_base(Self::default_url(), cache, reload, cache_setting) + Self::from_base( + Self::default_url(), + cache, + reload, + cache_setting, + progress_bar, + ) } pub fn from_base( @@ -141,6 +150,7 @@ impl NpmRegistryApi { cache: NpmCache, reload: bool, cache_setting: CacheSetting, + progress_bar: ProgressBar, ) -> Self { Self { base_url, @@ -148,6 +158,7 @@ impl NpmRegistryApi { mem_cache: Default::default(), reload, cache_setting, + progress_bar, } } @@ -294,13 +305,7 @@ impl NpmRegistryApi { } let package_url = self.get_package_url(name); - - log::log!( - log::Level::Info, - "{} {}", - colors::green("Download"), - package_url, - ); + let _guard = self.progress_bar.update(package_url.as_str()); let response = match reqwest::get(package_url).await { Ok(response) => response, diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 4a8deb1caa..f19132c8b2 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -25,6 +25,7 @@ use crate::node::NodeResolution; use crate::npm::GlobalNpmPackageResolver; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageResolver; +use crate::progress_bar::ProgressBar; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; use crate::tools::check; @@ -88,6 +89,7 @@ pub struct Inner { maybe_file_watcher_reporter: Option, pub npm_resolver: GlobalNpmPackageResolver, pub cjs_resolutions: Mutex>, + progress_bar: ProgressBar, } impl Deref for ProcState { @@ -147,6 +149,7 @@ impl ProcState { let http_cache = http_cache::HttpCache::new(&deps_cache_location); let root_cert_store = cli_options.resolve_root_cert_store()?; let cache_usage = cli_options.cache_setting(); + let progress_bar = ProgressBar::default(); let file_fetcher = FileFetcher::new( http_cache, cache_usage, @@ -156,6 +159,7 @@ impl ProcState { cli_options .unsafely_ignore_certificate_errors() .map(ToOwned::to_owned), + Some(progress_bar.clone()), )?; let lockfile = cli_options @@ -224,6 +228,7 @@ impl ProcState { // don't do the unstable error when in the lsp || matches!(cli_options.sub_command(), DenoSubcommand::Lsp), cli_options.no_npm(), + progress_bar.clone(), ); let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into(); @@ -250,6 +255,7 @@ impl ProcState { maybe_file_watcher_reporter, npm_resolver, cjs_resolutions: Default::default(), + progress_bar, }))) } @@ -411,6 +417,8 @@ impl ProcState { self.prepare_node_std_graph().await?; } + self.progress_bar.clear(); + // type check if necessary if self.options.type_check_mode() != TypeCheckMode::None { let maybe_config_specifier = self.options.maybe_config_file_specifier(); diff --git a/cli/progress_bar.rs b/cli/progress_bar.rs new file mode 100644 index 0000000000..f52d137c2b --- /dev/null +++ b/cli/progress_bar.rs @@ -0,0 +1,132 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use crate::colors; +use deno_core::parking_lot::Mutex; +use indexmap::IndexSet; +use std::sync::Arc; +use std::time::Duration; + +#[derive(Clone, Debug, Default)] +pub struct ProgressBar(Arc>); + +#[derive(Debug)] +struct ProgressBarInner { + pb: Option, + is_tty: bool, + in_flight: IndexSet, +} + +impl Default for ProgressBarInner { + fn default() -> Self { + Self { + pb: None, + is_tty: colors::is_tty(), + in_flight: IndexSet::default(), + } + } +} + +impl ProgressBarInner { + fn get_or_create_pb(&mut self) -> indicatif::ProgressBar { + if let Some(pb) = self.pb.as_ref() { + return pb.clone(); + } + + let pb = indicatif::ProgressBar::new_spinner(); + pb.enable_steady_tick(Duration::from_millis(120)); + pb.set_prefix("Download"); + pb.set_style( + indicatif::ProgressStyle::with_template( + "{prefix:.green} {spinner:.green} {msg}", + ) + .unwrap() + .tick_strings(&["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]), + ); + self.pb = Some(pb); + self.pb.as_ref().unwrap().clone() + } + + fn add_in_flight(&mut self, msg: &str) { + if self.in_flight.contains(msg) { + return; + } + + self.in_flight.insert(msg.to_string()); + } + + /// Returns if removed "in-flight" was last entry and progress + /// bar needs to be updated. + fn remove_in_flight(&mut self, msg: &str) -> bool { + if !self.in_flight.contains(msg) { + return false; + } + + let mut is_last = false; + if let Some(last) = self.in_flight.last() { + is_last = last == msg; + } + self.in_flight.remove(msg); + is_last + } + + fn update_progress_bar(&mut self) { + let pb = self.get_or_create_pb(); + if let Some(msg) = self.in_flight.last() { + pb.set_message(msg.clone()); + } + } +} + +pub struct UpdateGuard { + pb: ProgressBar, + msg: String, + noop: bool, +} + +impl Drop for UpdateGuard { + fn drop(&mut self) { + if self.noop { + return; + } + + let mut inner = self.pb.0.lock(); + if inner.remove_in_flight(&self.msg) { + inner.update_progress_bar(); + } + } +} + +impl ProgressBar { + pub fn update(&self, msg: &str) -> UpdateGuard { + let mut guard = UpdateGuard { + pb: self.clone(), + msg: msg.to_string(), + noop: false, + }; + let mut inner = self.0.lock(); + + // If we're not running in TTY we're just gonna fallback + // to using logger crate. + if !inner.is_tty { + log::log!(log::Level::Info, "{} {}", colors::green("Download"), msg); + guard.noop = true; + return guard; + } + + inner.add_in_flight(msg); + inner.update_progress_bar(); + guard + } + + pub fn clear(&self) { + let mut inner = self.0.lock(); + + match inner.pb.as_ref() { + Some(pb) => { + pb.finish_and_clear(); + inner.pb = None; + } + None => {} + }; + } +}