Add more async module loading function (#1974)

* get_source_code_async
* fetch_module_meta_data_async
* fetch_module_meta_data_and_maybe_compile_async

cli/deno_dir.rs

@@ -11,6 +11,7 @@ use crate::msg;
 use crate::tokio_util;
 use crate::version;
 use dirs;
+use futures::future::Either;
 use futures::Future;
 use ring;
 use std;
@@ -117,22 +118,6 @@ impl DenoDir {
     )
   }
 
-  fn load_cache(
-    self: &Self,
-    filename: &str,
-    source_code: &[u8],
-  ) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
-    let (output_code, source_map) = self.cache_path(filename, source_code);
-    debug!(
-      "load_cache code: {} map: {}",
-      output_code.display(),
-      source_map.display()
-    );
-    let read_output_code = fs::read(&output_code)?;
-    let read_source_map = fs::read(&source_map)?;
-    Ok((read_output_code, read_source_map))
-  }
-
   pub fn code_cache(
     self: &Self,
     module_meta_data: &ModuleMetaData,
@@ -158,13 +143,14 @@ impl DenoDir {
     }
   }
 
-  // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
+  fn get_source_code_async(
-  fn get_source_code(
     self: &Self,
     module_name: &str,
     filename: &str,
-  ) -> DenoResult<ModuleMetaData> {
+  ) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
-    let is_module_remote = is_remote(module_name);
+    let filename = filename.to_string();
+    let module_name = module_name.to_string();
+    let is_module_remote = is_remote(&module_name);
     // We try fetch local. Two cases:
     // 1. This is a remote module, but no reload provided
     // 2. This is a local module
@@ -173,114 +159,165 @@ impl DenoDir {
         "fetch local or reload {} is_module_remote {}",
         module_name, is_module_remote
       );
-      match fetch_local_source(&module_name, &filename)? {
+      // Note that local fetch is done synchronously.
-        Some(output) => {
+      match fetch_local_source(&module_name, &filename) {
+        Ok(Some(output)) => {
           debug!("found local source ");
-          return Ok(output);
+          return Either::A(futures::future::ok(output));
         }
-        None => {
+        Ok(None) => {
           debug!("fetch_local_source returned None");
         }
+        Err(err) => {
+          return Either::A(futures::future::err(err));
+        }
       }
     }
 
     // If not remote file, stop here!
     if !is_module_remote {
       debug!("not remote file stop here");
-      return Err(DenoError::from(std::io::Error::new(
+      return Either::A(futures::future::err(DenoError::from(
-        std::io::ErrorKind::NotFound,
+        std::io::Error::new(
-        format!("cannot find local file '{}'", filename),
+          std::io::ErrorKind::NotFound,
+          format!("cannot find local file '{}'", &filename),
+        ),
       )));
     }
 
     debug!("is remote but didn't find module");
 
+    let filename2 = filename.clone();
     // not cached/local, try remote
-    let maybe_remote_source = fetch_remote_source(&module_name, &filename)?;
+    let f = fetch_remote_source_async(&module_name, &filename).and_then(
-    if let Some(output) = maybe_remote_source {
+      move |maybe_remote_source| match maybe_remote_source {
-      return Ok(output);
+        Some(output) => Ok(output),
-    }
+        None => Err(DenoError::from(std::io::Error::new(
-    Err(DenoError::from(std::io::Error::new(
+          std::io::ErrorKind::NotFound,
-      std::io::ErrorKind::NotFound,
+          format!("cannot find remote file '{}'", &filename2),
-      format!("cannot find remote file '{}'", filename),
+        ))),
-    )))
+      },
+    );
+    Either::B(f)
   }
 
-  pub fn fetch_module_meta_data(
+  #[cfg(test)]
+  /// Synchronous version of get_source_code_async
+  /// This function is deprecated.
+  fn get_source_code(
+    self: &Self,
+    module_name: &str,
+    filename: &str,
+  ) -> DenoResult<ModuleMetaData> {
+    tokio_util::block_on(self.get_source_code_async(module_name, filename))
+  }
+
+  pub fn fetch_module_meta_data_async(
     self: &Self,
     specifier: &str,
     referrer: &str,
-  ) -> Result<ModuleMetaData, errors::DenoError> {
+  ) -> impl Future<Item = ModuleMetaData, Error = errors::DenoError> {
     debug!(
       "fetch_module_meta_data. specifier {} referrer {}",
       specifier, referrer
     );
 
-    let (module_name, filename) = self.resolve_module(specifier, referrer)?;
+    let specifier = specifier.to_string();
+    let referrer = referrer.to_string();
 
-    let result = self.get_source_code(module_name.as_str(), filename.as_str());
+    let result = self.resolve_module(&specifier, &referrer);
-    let mut out = match result {
+    if let Err(err) = result {
-      Ok(out) => out,
+      return Either::A(futures::future::err(DenoError::from(err)));
-      Err(err) => {
-        if err.kind() == ErrorKind::NotFound {
-          // For NotFound, change the message to something better.
-          return Err(errors::new(
-            ErrorKind::NotFound,
-            format!(
-              "Cannot resolve module \"{}\" from \"{}\"",
-              specifier, referrer
-            ),
-          ));
-        } else {
-          return Err(err);
-        }
-      }
-    };
-
-    if out.source_code.starts_with("#!".as_bytes()) {
-      out.source_code = filter_shebang(out.source_code);
     }
+    let (module_name, filename) = result.unwrap();
 
-    if out.media_type != msg::MediaType::TypeScript {
+    let gen = self.gen.clone();
-      return Ok(out);
+    let recompile = self.recompile;
-    }
 
-    let (output_code_filename, output_source_map_filename) =
+    Either::B(
-      self.cache_path(&out.filename, &out.source_code);
+      self
-    let mut maybe_output_code = None;
+        .get_source_code_async(module_name.as_str(), filename.as_str())
-    let mut maybe_source_map = None;
+        .then(move |result| {
+          let mut out = match result {
+            Ok(out) => out,
+            Err(err) => {
+              if err.kind() == ErrorKind::NotFound {
+                // For NotFound, change the message to something better.
+                return Err(errors::new(
+                  ErrorKind::NotFound,
+                  format!(
+                    "Cannot resolve module \"{}\" from \"{}\"",
+                    specifier, referrer
+                  ),
+                ));
+              } else {
+                return Err(err);
+              }
+            }
+          };
 
-    if !self.recompile {
+          if out.source_code.starts_with("#!".as_bytes()) {
-      let result = self.load_cache(out.filename.as_str(), &out.source_code);
+            out.source_code = filter_shebang(out.source_code);
-      match result {
-        Err(err) => {
-          if err.kind() == std::io::ErrorKind::NotFound {
-            return Ok(out);
-          } else {
-            return Err(err.into());
           }
-        }
-        Ok((output_code, source_map)) => {
-          maybe_output_code = Some(output_code);
-          maybe_source_map = Some(source_map);
-        }
-      }
-    }
 
-    Ok(ModuleMetaData {
+          if out.media_type != msg::MediaType::TypeScript {
-      module_name: out.module_name,
+            return Ok(out);
-      filename: out.filename,
+          }
-      media_type: out.media_type,
+
-      source_code: out.source_code,
+          let cache_key =
-      maybe_output_code_filename: output_code_filename
+            source_code_hash(&out.filename, &out.source_code, version::DENO);
-        .to_str()
+          let (output_code_filename, output_source_map_filename) = (
-        .map(String::from),
+            gen.join(cache_key.to_string() + ".js"),
-      maybe_output_code,
+            gen.join(cache_key.to_string() + ".js.map"),
-      maybe_source_map_filename: output_source_map_filename
+          );
-        .to_str()
+
-        .map(String::from),
+          let mut maybe_output_code = None;
-      maybe_source_map,
+          let mut maybe_source_map = None;
-    })
+
+          if !recompile {
+            let result =
+              load_cache2(&output_code_filename, &output_source_map_filename);
+            match result {
+              Err(err) => {
+                if err.kind() == std::io::ErrorKind::NotFound {
+                  return Ok(out);
+                } else {
+                  return Err(err.into());
+                }
+              }
+              Ok((output_code, source_map)) => {
+                maybe_output_code = Some(output_code);
+                maybe_source_map = Some(source_map);
+              }
+            }
+          }
+
+          Ok(ModuleMetaData {
+            module_name: out.module_name,
+            filename: out.filename,
+            media_type: out.media_type,
+            source_code: out.source_code,
+            maybe_output_code_filename: output_code_filename
+              .to_str()
+              .map(String::from),
+            maybe_output_code,
+            maybe_source_map_filename: output_source_map_filename
+              .to_str()
+              .map(String::from),
+            maybe_source_map,
+          })
+        }),
+    )
+  }
+
+  /// Synchronous version of fetch_module_meta_data_async
+  /// This function is deprecated.
+  pub fn fetch_module_meta_data(
+    self: &Self,
+    specifier: &str,
+    referrer: &str,
+  ) -> Result<ModuleMetaData, errors::DenoError> {
+    tokio_util::block_on(self.fetch_module_meta_data_async(specifier, referrer))
   }
 
   // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L56-L68
@@ -409,6 +446,20 @@ fn get_cache_filename(basedir: &Path, url: &Url) -> PathBuf {
   out
 }
 
+fn load_cache2(
+  js_filename: &PathBuf,
+  map_filename: &PathBuf,
+) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
+  debug!(
+    "load_cache code: {} map: {}",
+    js_filename.display(),
+    map_filename.display()
+  );
+  let read_output_code = fs::read(&js_filename)?;
+  let read_source_map = fs::read(&map_filename)?;
+  Ok((read_output_code, read_source_map))
+}
+
 fn source_code_hash(
   filename: &str,
   source_code: &[u8],
@@ -546,6 +597,7 @@ fn fetch_remote_source_async(
 
 // Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
 /// Fetch remote source code.
+#[cfg(test)]
 fn fetch_remote_source(
   module_name: &str,
   filename: &str,
@@ -988,19 +1040,21 @@ mod tests {
     let cwd = std::env::current_dir().unwrap();
     let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
 
-    // Test failure case.
+    tokio_util::init(|| {
-    let specifier = "hello.ts";
+      // Test failure case.
-    let referrer = add_root!("/baddir/badfile.ts");
+      let specifier = "hello.ts";
-    let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      let referrer = add_root!("/baddir/badfile.ts");
-    assert!(r.is_err());
+      let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      assert!(r.is_err());
 
-    // Assuming cwd is the deno repo root.
+      // Assuming cwd is the deno repo root.
-    let specifier = "./js/main.ts";
+      let specifier = "./js/main.ts";
-    let referrer = cwd_string.as_str();
+      let referrer = cwd_string.as_str();
-    let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      let r = deno_dir.fetch_module_meta_data(specifier, referrer);
-    assert!(r.is_ok());
+      assert!(r.is_ok());
-    //let fetch_module_meta_data_output = r.unwrap();
+      //let fetch_module_meta_data_output = r.unwrap();
-    //println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
+      //println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
+    })
   }
 
   #[test]
@@ -1011,17 +1065,19 @@ mod tests {
     let cwd = std::env::current_dir().unwrap();
     let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
 
-    // Test failure case.
+    tokio_util::init(|| {
-    let specifier = "hello.ts";
+      // Test failure case.
-    let referrer = add_root!("/baddir/badfile.ts");
+      let specifier = "hello.ts";
-    let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      let referrer = add_root!("/baddir/badfile.ts");
-    assert!(r.is_err());
+      let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      assert!(r.is_err());
 
-    // Assuming cwd is the deno repo root.
+      // Assuming cwd is the deno repo root.
-    let specifier = "./js/main.ts";
+      let specifier = "./js/main.ts";
-    let referrer = cwd_string.as_str();
+      let referrer = cwd_string.as_str();
-    let r = deno_dir.fetch_module_meta_data(specifier, referrer);
+      let r = deno_dir.fetch_module_meta_data(specifier, referrer);
-    assert!(r.is_ok());
+      assert!(r.is_ok());
+    })
   }
 
   #[test]

cli/isolate.rs

@@ -7,6 +7,7 @@ use crate::isolate_state::IsolateState;
 use crate::isolate_state::IsolateStateContainer;
 use crate::js_errors;
 use crate::msg;
+use crate::tokio_util;
 use deno_core;
 use deno_core::deno_mod;
 use deno_core::Behavior;
@@ -162,23 +163,39 @@ impl<B: DenoBehavior> Future for Isolate<B> {
     self.inner.poll().map_err(|err| self.apply_source_map(err))
   }
 }
+fn fetch_module_meta_data_and_maybe_compile_async(
+  state: &Arc<IsolateState>,
+  specifier: &str,
+  referrer: &str,
+) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
+  let state_ = state.clone();
+  let specifier = specifier.to_string();
+  let referrer = referrer.to_string();
+  state
+    .dir
+    .fetch_module_meta_data_async(&specifier, &referrer)
+    .and_then(move |mut out| {
+      if (out.media_type == msg::MediaType::TypeScript
+        && out.maybe_output_code.is_none())
+        || state_.flags.recompile
+      {
+        debug!(">>>>> compile_sync START");
+        out = compile_sync(state_.clone(), &specifier, &referrer, &out);
+        debug!(">>>>> compile_sync END");
+        state_.dir.code_cache(&out)?;
+      }
+      Ok(out)
+    })
+}
 
 fn fetch_module_meta_data_and_maybe_compile(
   state: &Arc<IsolateState>,
   specifier: &str,
   referrer: &str,
 ) -> Result<ModuleMetaData, DenoError> {
-  let mut out = state.dir.fetch_module_meta_data(specifier, referrer)?;
+  tokio_util::block_on(fetch_module_meta_data_and_maybe_compile_async(
-  if (out.media_type == msg::MediaType::TypeScript
+    state, specifier, referrer,
-    && out.maybe_output_code.is_none())
+  ))
-    || state.flags.recompile
-  {
-    debug!(">>>>> compile_sync START");
-    out = compile_sync(state.clone(), specifier, &referrer, &out);
-    debug!(">>>>> compile_sync END");
-    state.dir.code_cache(&out)?;
-  }
-  Ok(out)
 }
 
 #[cfg(test)]
@@ -186,7 +203,6 @@ mod tests {
   use super::*;
   use crate::cli_behavior::CliBehavior;
   use crate::flags;
-  use crate::tokio_util;
   use futures::future::lazy;
   use std::sync::atomic::Ordering;
 

cli/ops.rs

@@ -1952,12 +1952,14 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
-    let fetch_result =
+    tokio_util::init(move || {
-      op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
+      let fetch_result =
-    match fetch_result {
+        op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
-      Ok(_) => assert!(true),
+      match fetch_result {
-      Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
+        Ok(_) => assert!(true),
-    }
+        Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
+      }
+    })
   }
 
   #[test]
@@ -1986,12 +1988,14 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
-    let fetch_result =
+    tokio_util::init(move || {
-      op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
+      let fetch_result =
-    match fetch_result {
+        op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
-      Ok(_) => assert!(true),
+      match fetch_result {
-      Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
+        Ok(_) => assert!(true),
-    }
+        Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
+      }
+    })
   }
 
   #[test]
@@ -2019,12 +2023,15 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
-    let fetch_result =
+
-      op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
+    tokio_util::init(move || {
-    match fetch_result {
+      let fetch_result =
-      Ok(_) => assert!(true),
+        op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
-      Err(e) => assert!(e.to_string() != permission_denied().to_string()),
+      match fetch_result {
-    }
+        Ok(_) => assert!(true),
+        Err(e) => assert!(e.to_string() != permission_denied().to_string()),
+      }
+    })
   }
 }
 */