Add progress bar (#2309)

cli/compiler.rs

@@ -95,6 +95,7 @@ fn lazy_start(parent_state: ThreadSafeState) -> ResourceId {
         parent_state.flags.clone(),
         parent_state.argv.clone(),
         op_selector_compiler,
+        parent_state.progress.clone(),
       );
       let rid = child_state.resource.rid;
       let resource = child_state.resource.clone();
@@ -192,6 +193,10 @@ pub fn compile_async(
 
   let compiler_rid = lazy_start(parent_state.clone());
 
+  let compiling_job = parent_state
+    .progress
+    .add(format!("Compiling {}", module_meta_data_.module_name));
+
   let (local_sender, local_receiver) =
     oneshot::channel::<Result<ModuleMetaData, Option<JSError>>>();
 
@@ -218,6 +223,10 @@ pub fn compile_async(
           let res_data = res["data"].as_object().expect(
             "Error decoding compiler response: expected object field 'data'",
           );
+
+          // Explicit drop to keep reference alive until future completes.
+          drop(compiling_job);
+
           match res["success"].as_bool() {
             Some(true) => Ok(ModuleMetaData {
               maybe_output_code: res_data["outputCode"]

cli/deno_dir.rs

@@ -8,6 +8,7 @@ use crate::fs as deno_fs;
 use crate::http_util;
 use crate::js_errors::SourceMapGetter;
 use crate::msg;
+use crate::progress::Progress;
 use crate::tokio_util;
 use crate::version;
 use dirs;
@@ -44,6 +45,8 @@ pub struct DenoDir {
   /// The active configuration file contents (or empty array) which applies to
   /// source code cached by `DenoDir`.
   pub config: Vec<u8>,
+
+  pub progress: Progress,
 }
 
 impl DenoDir {
@@ -52,6 +55,7 @@ impl DenoDir {
   pub fn new(
     custom_root: Option<PathBuf>,
     state_config: &Option<Vec<u8>>,
+    progress: Progress,
   ) -> std::io::Result<Self> {
     // Only setup once.
     let home_dir = dirs::home_dir().expect("Could not get home directory.");
@@ -85,6 +89,7 @@ impl DenoDir {
       deps_http,
       deps_https,
       config,
+      progress,
     };
 
     // TODO Lazily create these directories.
@@ -578,9 +583,10 @@ fn fetch_remote_source_async(
   filename: &str,
 ) -> impl Future<Item = Option<ModuleMetaData>, Error = DenoError> {
   use crate::http_util::FetchOnceResult;
-  {
+
-    eprintln!("Downloading {}", module_name);
+  let download_job = deno_dir
-  }
+    .progress
+    .add(format!("Downloading {}", module_name));
 
   let filename = filename.to_owned();
   let module_name = module_name.to_owned();
@@ -682,7 +688,11 @@ fn fetch_remote_source_async(
         }
       })
     },
-  )
+  ).then(move |r| {
+    // Explicit drop to keep reference alive until future completes.
+    drop(download_job);
+    r
+  })
 }
 
 /// Fetch remote source code.
@@ -912,8 +922,11 @@ mod tests {
   fn test_setup() -> (TempDir, DenoDir) {
     let temp_dir = TempDir::new().expect("tempdir fail");
     let config = Some(b"{}".to_vec());
-    let deno_dir = DenoDir::new(Some(temp_dir.path().to_path_buf()), &config)
+    let deno_dir = DenoDir::new(
-      .expect("setup fail");
+      Some(temp_dir.path().to_path_buf()),
+      &config,
+      Progress::new(),
+    ).expect("setup fail");
     (temp_dir, deno_dir)
   }
 

cli/flags.rs

@@ -7,7 +7,6 @@ macro_rules! svec {
     ($($x:expr),*) => (vec![$($x.to_string()),*]);
 }
 
-#[cfg_attr(feature = "cargo-clippy", allow(stutter))]
 #[derive(Clone, Debug, PartialEq, Default)]
 pub struct DenoFlags {
   pub log_debug: bool,
@@ -307,7 +306,6 @@ fn resolve_paths(paths: Vec<String>) -> Vec<String> {
 
 /// Parse ArgMatches into internal DenoFlags structure.
 /// This method should not make any side effects.
-#[cfg_attr(feature = "cargo-clippy", allow(stutter))]
 pub fn parse_flags(matches: ArgMatches) -> DenoFlags {
   let mut flags = DenoFlags::default();
 

cli/main.rs

@@ -27,6 +27,7 @@ pub mod msg;
 pub mod msg_util;
 pub mod ops;
 pub mod permissions;
+mod progress;
 mod repl;
 pub mod resolve_addr;
 pub mod resources;
@@ -39,6 +40,7 @@ pub mod version;
 pub mod worker;
 
 use crate::errors::RustOrJsError;
+use crate::progress::Progress;
 use crate::state::ThreadSafeState;
 use crate::worker::root_specifier_to_url;
 use crate::worker::Worker;
@@ -134,7 +136,16 @@ fn create_worker_and_state(
   flags: DenoFlags,
   argv: Vec<String>,
 ) -> (Worker, ThreadSafeState) {
-  let state = ThreadSafeState::new(flags, argv, ops::op_selector_std);
+  let progress = Progress::new();
+  progress.set_callback(|done, completed, total, msg| {
+    if done {
+      eprintln!("");
+    } else {
+      eprint!("\r[{}/{}] {}", completed, total, msg);
+      eprint!("\x1B[K"); // Clear to end of line.
+    }
+  });
+  let state = ThreadSafeState::new(flags, argv, ops::op_selector_std, progress);
   let worker = Worker::new(
     "main".to_string(),
     startup_data::deno_isolate_init(),

cli/msg.rs

@@ -1,5 +1,4 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-#![allow(unused_imports)]
 #![allow(dead_code)]
 #![cfg_attr(
   feature = "cargo-clippy",

cli/ops.rs

@@ -859,19 +859,9 @@ fn op_chmod(
     debug!("op_chmod {}", &path_);
     // Still check file/dir exists on windows
     let _metadata = fs::metadata(&path)?;
-    // Only work in unix
     #[cfg(any(unix))]
     {
-      // We need to use underscore to compile in Windows.
-      #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::used_underscore_binding)
-      )]
       let mut permissions = _metadata.permissions();
-      #[cfg_attr(
-        feature = "cargo-clippy",
-        allow(clippy::used_underscore_binding)
-      )]
       permissions.set_mode(_mode);
       fs::set_permissions(&path, permissions)?;
     }
@@ -2049,6 +2039,7 @@ fn op_create_worker(
       parent_state.flags.clone(),
       parent_state.argv.clone(),
       op_selector_std,
+      parent_state.progress.clone(),
     );
     let rid = child_state.resource.rid;
     let name = format!("USER-WORKER-{}", specifier);

cli/permissions.rs

@@ -124,7 +124,6 @@ impl Default for PermissionAccessor {
   }
 }
 
-#[cfg_attr(feature = "cargo-clippy", allow(stutter))]
 #[derive(Debug, Default)]
 pub struct DenoPermissions {
   // Keep in sync with src/permissions.ts

cli/progress.rs

@@ -0,0 +1,161 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use std::sync::Arc;
+use std::sync::Mutex;
+
+#[derive(Clone, Default)]
+pub struct Progress(Arc<Mutex<Inner>>);
+
+impl Progress {
+  pub fn new() -> Self {
+    Progress::default()
+  }
+
+  pub fn set_callback<F>(&self, f: F)
+  where
+    F: Fn(bool, usize, usize, &str) + Send + Sync + 'static,
+  {
+    let mut s = self.0.lock().unwrap();
+    assert!(s.callback.is_none());
+    s.callback = Some(Arc::new(f));
+  }
+
+  /// Returns job counts: (complete, total)
+  pub fn progress(&self) -> (usize, usize) {
+    let s = self.0.lock().unwrap();
+    s.progress()
+  }
+
+  pub fn history(&self) -> Vec<String> {
+    let s = self.0.lock().unwrap();
+    s.job_names.clone()
+  }
+
+  pub fn add(&self, name: String) -> Job {
+    let mut s = self.0.lock().unwrap();
+    let id = s.job_names.len();
+    s.maybe_call_callback(false, s.complete, s.job_names.len() + 1, &name);
+    s.job_names.push(name);
+    Job {
+      id,
+      inner: self.0.clone(),
+    }
+  }
+
+  pub fn done(&self) {
+    let s = self.0.lock().unwrap();
+    s.maybe_call_callback(true, s.complete, s.job_names.len(), "");
+  }
+}
+
+type Callback = dyn Fn(bool, usize, usize, &str) + Send + Sync;
+
+#[derive(Default)]
+struct Inner {
+  job_names: Vec<String>,
+  complete: usize,
+  callback: Option<Arc<Callback>>,
+}
+
+impl Inner {
+  pub fn maybe_call_callback(
+    &self,
+    done: bool,
+    complete: usize,
+    total: usize,
+    msg: &str,
+  ) {
+    if let Some(ref cb) = self.callback {
+      cb(done, complete, total, msg);
+    }
+  }
+
+  /// Returns job counts: (complete, total)
+  pub fn progress(&self) -> (usize, usize) {
+    let total = self.job_names.len();
+    (self.complete, total)
+  }
+}
+
+pub struct Job {
+  inner: Arc<Mutex<Inner>>,
+  id: usize,
+}
+
+impl Drop for Job {
+  fn drop(&mut self) {
+    let mut s = self.inner.lock().unwrap();
+    s.complete += 1;
+    let name = &s.job_names[self.id];
+    let (complete, total) = s.progress();
+    s.maybe_call_callback(false, complete, total, name);
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn progress() {
+    let p = Progress::new();
+    assert_eq!(p.progress(), (0, 0));
+    {
+      let _j1 = p.add("hello".to_string());
+      assert_eq!(p.progress(), (0, 1));
+    }
+    assert_eq!(p.progress(), (1, 1));
+    {
+      let _j2 = p.add("hello".to_string());
+      assert_eq!(p.progress(), (1, 2));
+    }
+    assert_eq!(p.progress(), (2, 2));
+  }
+
+  #[test]
+  fn history() {
+    let p = Progress::new();
+    let _a = p.add("a".to_string());
+    let _b = p.add("b".to_string());
+    assert_eq!(p.history(), vec!["a", "b"]);
+  }
+
+  #[test]
+  fn callback() {
+    let callback_history: Arc<Mutex<Vec<(usize, usize, String)>>> =
+      Arc::new(Mutex::new(Vec::new()));
+    {
+      let p = Progress::new();
+      let callback_history_ = callback_history.clone();
+
+      p.set_callback(move |_done, complete, total, msg| {
+        // println!("callback: {}, {}, {}", complete, total, msg);
+        let mut h = callback_history_.lock().unwrap();
+        h.push((complete, total, String::from(msg)));
+      });
+      {
+        let _a = p.add("a".to_string());
+        let _b = p.add("b".to_string());
+      }
+      let _c = p.add("c".to_string());
+    }
+
+    let h = callback_history.lock().unwrap();
+    assert_eq!(
+      h.to_vec(),
+      vec![
+        (0, 1, "a".to_string()),
+        (0, 2, "b".to_string()),
+        (1, 2, "b".to_string()),
+        (2, 2, "a".to_string()),
+        (2, 3, "c".to_string()),
+        (3, 3, "c".to_string()),
+      ]
+    );
+  }
+
+  #[test]
+  fn thread_safe() {
+    fn f<S: Send + Sync>(_: S) {}
+    f(Progress::new());
+  }
+}

cli/resources.rs

@@ -374,7 +374,6 @@ pub fn get_message_stream_from_worker(rid: ResourceId) -> WorkerReceiverStream {
   WorkerReceiverStream { rid }
 }
 
-#[cfg_attr(feature = "cargo-clippy", allow(stutter))]
 pub struct ChildResources {
   pub child_rid: ResourceId,
   pub stdin_rid: Option<ResourceId>,

cli/state.rs

@@ -5,6 +5,7 @@ use crate::flags;
 use crate::global_timer::GlobalTimer;
 use crate::ops;
 use crate::permissions::DenoPermissions;
+use crate::progress::Progress;
 use crate::resources;
 use crate::resources::ResourceId;
 use crate::worker::Worker;
@@ -28,7 +29,6 @@ pub type WorkerReceiver = async_mpsc::Receiver<Buf>;
 pub type WorkerChannels = (WorkerSender, WorkerReceiver);
 pub type UserWorkerTable = HashMap<ResourceId, Shared<Worker>>;
 
-// AtomicU64 is currently unstable
 #[derive(Default)]
 pub struct Metrics {
   pub ops_dispatched: AtomicUsize,
@@ -39,13 +39,11 @@ pub struct Metrics {
   pub resolve_count: AtomicUsize,
 }
 
-// Wrap State so that it can implement Dispatch.
+/// Isolate cannot be passed between threads but ThreadSafeState can.
+/// ThreadSafeState satisfies Send and Sync. So any state that needs to be
+/// accessed outside the main V8 thread should be inside ThreadSafeState.
 pub struct ThreadSafeState(Arc<State>);
 
-// Isolate cannot be passed between threads but ThreadSafeState can.
-// ThreadSafeState satisfies Send and Sync.
-// So any state that needs to be accessed outside the main V8 thread should be
-// inside ThreadSafeState.
 #[cfg_attr(feature = "cargo-clippy", allow(stutter))]
 pub struct State {
   pub dir: deno_dir::DenoDir,
@@ -63,8 +61,11 @@ pub struct State {
   pub global_timer: Mutex<GlobalTimer>,
   pub workers: Mutex<UserWorkerTable>,
   pub start_time: Instant,
+  /// A reference to this worker's resource.
   pub resource: resources::Resource,
   pub dispatch_selector: ops::OpSelector,
+  /// Reference to global progress bar.
+  pub progress: Progress,
 }
 
 impl Clone for ThreadSafeState {
@@ -91,6 +92,7 @@ impl ThreadSafeState {
     flags: flags::DenoFlags,
     argv_rest: Vec<String>,
     dispatch_selector: ops::OpSelector,
+    progress: Progress,
   ) -> Self {
     let custom_root = env::var("DENO_DIR").map(String::into).ok();
 
@@ -140,7 +142,8 @@ impl ThreadSafeState {
     };
 
     ThreadSafeState(Arc::new(State {
-      dir: deno_dir::DenoDir::new(custom_root, &config).unwrap(),
+      dir: deno_dir::DenoDir::new(custom_root, &config, progress.clone())
+        .unwrap(),
       argv: argv_rest,
       permissions: DenoPermissions::from_flags(&flags),
       flags,
@@ -153,6 +156,7 @@ impl ThreadSafeState {
       start_time: Instant::now(),
       resource,
       dispatch_selector,
+      progress,
     }))
   }
 
@@ -210,6 +214,7 @@ impl ThreadSafeState {
       flags::DenoFlags::default(),
       argv,
       ops::op_selector_std,
+      Progress::new(),
     )
   }
 

cli/worker.rs

@@ -69,6 +69,7 @@ impl Worker {
     let recursive_load = deno::RecursiveLoad::new(js_url.as_str(), self);
     recursive_load.and_then(
       move |(id, mut self_)| -> Result<Self, (deno::JSErrorOr<DenoError>, Self)> {
+        self_.state.progress.done();
         if is_prefetch {
           Ok(self_)
         } else {
@@ -82,6 +83,7 @@ impl Worker {
       },
     )
     .map_err(|(err, self_)| {
+      self_.state.progress.done();
       // Convert to RustOrJsError AND apply_source_map.
       let err = match err {
         deno::JSErrorOr::JSError(err) => RustOrJsError::Js(self_.apply_source_map(err)),
@@ -256,6 +258,7 @@ mod tests {
   use super::*;
   use crate::flags;
   use crate::ops::op_selector_std;
+  use crate::progress::Progress;
   use crate::resources;
   use crate::startup_data;
   use crate::state::ThreadSafeState;
@@ -272,8 +275,12 @@ mod tests {
     let js_url = Url::from_file_path(filename).unwrap();
 
     let argv = vec![String::from("./deno"), js_url.to_string()];
-    let state =
+    let state = ThreadSafeState::new(
-      ThreadSafeState::new(flags::DenoFlags::default(), argv, op_selector_std);
+      flags::DenoFlags::default(),
+      argv,
+      op_selector_std,
+      Progress::new(),
+    );
     let state_ = state.clone();
     tokio_util::run(lazy(move || {
       let worker = Worker::new("TEST".to_string(), StartupData::None, state);
@@ -298,8 +305,12 @@ mod tests {
     let js_url = Url::from_file_path(filename).unwrap();
 
     let argv = vec![String::from("./deno"), js_url.to_string()];
-    let state =
+    let state = ThreadSafeState::new(
-      ThreadSafeState::new(flags::DenoFlags::default(), argv, op_selector_std);
+      flags::DenoFlags::default(),
+      argv,
+      op_selector_std,
+      Progress::new(),
+    );
     let state_ = state.clone();
     tokio_util::run(lazy(move || {
       let worker = Worker::new("TEST".to_string(), StartupData::None, state);
@@ -318,6 +329,40 @@ mod tests {
     assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 2);
   }
 
+  #[test]
+  fn execute_006_url_imports() {
+    let filename = std::env::current_dir()
+      .unwrap()
+      .join("tests/006_url_imports.ts");
+    let js_url = Url::from_file_path(filename).unwrap();
+    let argv = vec![String::from("deno"), js_url.to_string()];
+    let mut flags = flags::DenoFlags::default();
+    flags.reload = true;
+    let state =
+      ThreadSafeState::new(flags, argv, op_selector_std, Progress::new());
+    let state_ = state.clone();
+    tokio_util::run(lazy(move || {
+      let mut worker = Worker::new(
+        "TEST".to_string(),
+        startup_data::deno_isolate_init(),
+        state,
+      );
+      js_check(worker.execute("denoMain()"));
+      let result = worker.execute_mod(&js_url, false);
+      let worker = match result {
+        Err((err, worker)) => {
+          eprintln!("execute_mod err {:?}", err);
+          worker
+        }
+        Ok(worker) => worker,
+      };
+      tokio_util::panic_on_error(worker)
+    }));
+
+    let metrics = &state_.metrics;
+    assert_eq!(metrics.resolve_count.load(Ordering::SeqCst), 3);
+  }
+
   fn create_test_worker() -> Worker {
     let state = ThreadSafeState::mock();
     let mut worker =

js/compiler.ts

@@ -400,9 +400,8 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
   ): { outputCode: OutputCode; sourceMap: SourceMap } {
     this._log("compiler.compile", { moduleSpecifier, containingFile });
     const moduleMetaData = this._resolveModule(moduleSpecifier, containingFile);
-    const { fileName, mediaType, moduleId, sourceCode } = moduleMetaData;
+    const { fileName, mediaType, sourceCode } = moduleMetaData;
     this._scriptFileNames = [fileName];
-    console.warn("Compiling", moduleId);
     let outputCode: string;
     let sourceMap = "";
     // Instead of using TypeScript to transpile JSON modules, we will just do

tests/config.ts.out

@@ -1,7 +1,7 @@
 Unsupported compiler options in "[WILDCARD]tests/config.tsconfig.json"
   The following options were ignored:
     module, target
-Compiling file://[WILDCARD]tests/config.ts
+
 [WILDCARD]tests/config.ts:3:5 - error TS2532: Object is possibly 'undefined'.
 
 3 if (map.get("bar").foo) {

tests/error_004_missing_module.ts.out

@@ -1,5 +1,4 @@
-Compiling [WILDCARD]tests/error_004_missing_module.ts
+[WILDCARD]Uncaught NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_004_missing_module.ts"
-Uncaught NotFound: Cannot resolve module "bad-module.ts" from "[WILDCARD]/tests/error_004_missing_module.ts"
     at DenoError (js/errors.ts:[WILDCARD])
     at maybeError (js/errors.ts:[WILDCARD])
     at maybeThrowError (js/errors.ts:[WILDCARD])

tests/error_006_import_ext_failure.ts.out

@@ -1,5 +1,4 @@
-Compiling [WILDCARD]tests/error_006_import_ext_failure.ts
+[WILDCARD]Uncaught NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests/error_006_import_ext_failure.ts"
-Uncaught NotFound: Cannot resolve module "./non-existent" from "[WILDCARD]/tests/error_006_import_ext_failure.ts"
     at DenoError (js/errors.ts:[WILDCARD])
     at maybeError (js/errors.ts:[WILDCARD])
     at maybeThrowError (js/errors.ts:[WILDCARD])

tests/unbuffered_stderr.ts.out

@@ -1,2 +1,2 @@
-Compiling [WILDCARD].ts
+[WILDCARD]
 x