Add more async module loading function (#1974)

* get_source_code_async
* fetch_module_meta_data_async
* fetch_module_meta_data_and_maybe_compile_async

cli/deno_dir.rs

@@ -11,6 +11,7 @@ use crate::msg;
 use crate::tokio_util;
 use crate::version;
 use dirs;
+use futures::future::Either;
 use futures::Future;
 use ring;
 use std;
@@ -117,22 +118,6 @@ impl DenoDir {
     )
   }
 
-  fn load_cache(
-    self: &Self,
-    filename: &str,
-    source_code: &[u8],
-  ) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
-    let (output_code, source_map) = self.cache_path(filename, source_code);
-    debug!(
-      "load_cache code: {} map: {}",
-      output_code.display(),
-      source_map.display()
-    );
-    let read_output_code = fs::read(&output_code)?;
-    let read_source_map = fs::read(&source_map)?;
-    Ok((read_output_code, read_source_map))
-  }
-
   pub fn code_cache(
     self: &Self,
     module_meta_data: &ModuleMetaData,
@@ -158,13 +143,14 @@ impl DenoDir {
     }
   }
 
-  // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L122-L138
-  fn get_source_code(
+  fn get_source_code_async(
     self: &Self,
     module_name: &str,
     filename: &str,
-  ) -> DenoResult<ModuleMetaData> {
-    let is_module_remote = is_remote(module_name);
+  ) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
+    let filename = filename.to_string();
+    let module_name = module_name.to_string();
+    let is_module_remote = is_remote(&module_name);
     // We try fetch local. Two cases:
     // 1. This is a remote module, but no reload provided
     // 2. This is a local module
@@ -173,52 +159,85 @@ impl DenoDir {
         "fetch local or reload {} is_module_remote {}",
         module_name, is_module_remote
       );
-      match fetch_local_source(&module_name, &filename)? {
-        Some(output) => {
+      // Note that local fetch is done synchronously.
+      match fetch_local_source(&module_name, &filename) {
+        Ok(Some(output)) => {
           debug!("found local source ");
-          return Ok(output);
+          return Either::A(futures::future::ok(output));
         }
-        None => {
+        Ok(None) => {
           debug!("fetch_local_source returned None");
         }
+        Err(err) => {
+          return Either::A(futures::future::err(err));
+        }
       }
     }
 
     // If not remote file, stop here!
     if !is_module_remote {
       debug!("not remote file stop here");
-      return Err(DenoError::from(std::io::Error::new(
+      return Either::A(futures::future::err(DenoError::from(
+        std::io::Error::new(
           std::io::ErrorKind::NotFound,
-        format!("cannot find local file '{}'", filename),
+          format!("cannot find local file '{}'", &filename),
+        ),
       )));
     }
 
     debug!("is remote but didn't find module");
 
+    let filename2 = filename.clone();
     // not cached/local, try remote
-    let maybe_remote_source = fetch_remote_source(&module_name, &filename)?;
-    if let Some(output) = maybe_remote_source {
-      return Ok(output);
-    }
-    Err(DenoError::from(std::io::Error::new(
+    let f = fetch_remote_source_async(&module_name, &filename).and_then(
+      move |maybe_remote_source| match maybe_remote_source {
+        Some(output) => Ok(output),
+        None => Err(DenoError::from(std::io::Error::new(
           std::io::ErrorKind::NotFound,
-      format!("cannot find remote file '{}'", filename),
-    )))
+          format!("cannot find remote file '{}'", &filename2),
+        ))),
+      },
+    );
+    Either::B(f)
   }
 
-  pub fn fetch_module_meta_data(
+  #[cfg(test)]
+  /// Synchronous version of get_source_code_async
+  /// This function is deprecated.
+  fn get_source_code(
+    self: &Self,
+    module_name: &str,
+    filename: &str,
+  ) -> DenoResult<ModuleMetaData> {
+    tokio_util::block_on(self.get_source_code_async(module_name, filename))
+  }
+
+  pub fn fetch_module_meta_data_async(
     self: &Self,
     specifier: &str,
     referrer: &str,
-  ) -> Result<ModuleMetaData, errors::DenoError> {
+  ) -> impl Future<Item = ModuleMetaData, Error = errors::DenoError> {
     debug!(
       "fetch_module_meta_data. specifier {} referrer {}",
       specifier, referrer
     );
 
-    let (module_name, filename) = self.resolve_module(specifier, referrer)?;
+    let specifier = specifier.to_string();
+    let referrer = referrer.to_string();
 
-    let result = self.get_source_code(module_name.as_str(), filename.as_str());
+    let result = self.resolve_module(&specifier, &referrer);
+    if let Err(err) = result {
+      return Either::A(futures::future::err(DenoError::from(err)));
+    }
+    let (module_name, filename) = result.unwrap();
+
+    let gen = self.gen.clone();
+    let recompile = self.recompile;
+
+    Either::B(
+      self
+        .get_source_code_async(module_name.as_str(), filename.as_str())
+        .then(move |result| {
           let mut out = match result {
             Ok(out) => out,
             Err(err) => {
@@ -245,13 +264,19 @@ impl DenoDir {
             return Ok(out);
           }
 
-    let (output_code_filename, output_source_map_filename) =
-      self.cache_path(&out.filename, &out.source_code);
+          let cache_key =
+            source_code_hash(&out.filename, &out.source_code, version::DENO);
+          let (output_code_filename, output_source_map_filename) = (
+            gen.join(cache_key.to_string() + ".js"),
+            gen.join(cache_key.to_string() + ".js.map"),
+          );
+
           let mut maybe_output_code = None;
           let mut maybe_source_map = None;
 
-    if !self.recompile {
-      let result = self.load_cache(out.filename.as_str(), &out.source_code);
+          if !recompile {
+            let result =
+              load_cache2(&output_code_filename, &output_source_map_filename);
             match result {
               Err(err) => {
                 if err.kind() == std::io::ErrorKind::NotFound {
@@ -281,6 +306,18 @@ impl DenoDir {
               .map(String::from),
             maybe_source_map,
           })
+        }),
+    )
+  }
+
+  /// Synchronous version of fetch_module_meta_data_async
+  /// This function is deprecated.
+  pub fn fetch_module_meta_data(
+    self: &Self,
+    specifier: &str,
+    referrer: &str,
+  ) -> Result<ModuleMetaData, errors::DenoError> {
+    tokio_util::block_on(self.fetch_module_meta_data_async(specifier, referrer))
   }
 
   // Prototype: https://github.com/denoland/deno/blob/golang/os.go#L56-L68
@@ -409,6 +446,20 @@ fn get_cache_filename(basedir: &Path, url: &Url) -> PathBuf {
   out
 }
 
+fn load_cache2(
+  js_filename: &PathBuf,
+  map_filename: &PathBuf,
+) -> Result<(Vec<u8>, Vec<u8>), std::io::Error> {
+  debug!(
+    "load_cache code: {} map: {}",
+    js_filename.display(),
+    map_filename.display()
+  );
+  let read_output_code = fs::read(&js_filename)?;
+  let read_source_map = fs::read(&map_filename)?;
+  Ok((read_output_code, read_source_map))
+}
+
 fn source_code_hash(
   filename: &str,
   source_code: &[u8],
@@ -546,6 +597,7 @@ fn fetch_remote_source_async(
 
 // Prototype https://github.com/denoland/deno/blob/golang/deno_dir.go#L37-L73
 /// Fetch remote source code.
+#[cfg(test)]
 fn fetch_remote_source(
   module_name: &str,
   filename: &str,
@@ -988,6 +1040,7 @@ mod tests {
     let cwd = std::env::current_dir().unwrap();
     let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
 
+    tokio_util::init(|| {
       // Test failure case.
       let specifier = "hello.ts";
       let referrer = add_root!("/baddir/badfile.ts");
@@ -1001,6 +1054,7 @@ mod tests {
       assert!(r.is_ok());
       //let fetch_module_meta_data_output = r.unwrap();
       //println!("fetch_module_meta_data_output {:?}", fetch_module_meta_data_output);
+    })
   }
 
   #[test]
@@ -1011,6 +1065,7 @@ mod tests {
     let cwd = std::env::current_dir().unwrap();
     let cwd_string = String::from(cwd.to_str().unwrap()) + "/";
 
+    tokio_util::init(|| {
       // Test failure case.
       let specifier = "hello.ts";
       let referrer = add_root!("/baddir/badfile.ts");
@@ -1022,6 +1077,7 @@ mod tests {
       let referrer = cwd_string.as_str();
       let r = deno_dir.fetch_module_meta_data(specifier, referrer);
       assert!(r.is_ok());
+    })
   }
 
   #[test]

cli/isolate.rs

@@ -7,6 +7,7 @@ use crate::isolate_state::IsolateState;
 use crate::isolate_state::IsolateStateContainer;
 use crate::js_errors;
 use crate::msg;
+use crate::tokio_util;
 use deno_core;
 use deno_core::deno_mod;
 use deno_core::Behavior;
@@ -162,23 +163,39 @@ impl<B: DenoBehavior> Future for Isolate<B> {
     self.inner.poll().map_err(|err| self.apply_source_map(err))
   }
 }
+fn fetch_module_meta_data_and_maybe_compile_async(
+  state: &Arc<IsolateState>,
+  specifier: &str,
+  referrer: &str,
+) -> impl Future<Item = ModuleMetaData, Error = DenoError> {
+  let state_ = state.clone();
+  let specifier = specifier.to_string();
+  let referrer = referrer.to_string();
+  state
+    .dir
+    .fetch_module_meta_data_async(&specifier, &referrer)
+    .and_then(move |mut out| {
+      if (out.media_type == msg::MediaType::TypeScript
+        && out.maybe_output_code.is_none())
+        || state_.flags.recompile
+      {
+        debug!(">>>>> compile_sync START");
+        out = compile_sync(state_.clone(), &specifier, &referrer, &out);
+        debug!(">>>>> compile_sync END");
+        state_.dir.code_cache(&out)?;
+      }
+      Ok(out)
+    })
+}
 
 fn fetch_module_meta_data_and_maybe_compile(
   state: &Arc<IsolateState>,
   specifier: &str,
   referrer: &str,
 ) -> Result<ModuleMetaData, DenoError> {
-  let mut out = state.dir.fetch_module_meta_data(specifier, referrer)?;
-  if (out.media_type == msg::MediaType::TypeScript
-    && out.maybe_output_code.is_none())
-    || state.flags.recompile
-  {
-    debug!(">>>>> compile_sync START");
-    out = compile_sync(state.clone(), specifier, &referrer, &out);
-    debug!(">>>>> compile_sync END");
-    state.dir.code_cache(&out)?;
-  }
-  Ok(out)
+  tokio_util::block_on(fetch_module_meta_data_and_maybe_compile_async(
+    state, specifier, referrer,
+  ))
 }
 
 #[cfg(test)]
@@ -186,7 +203,6 @@ mod tests {
   use super::*;
   use crate::cli_behavior::CliBehavior;
   use crate::flags;
-  use crate::tokio_util;
   use futures::future::lazy;
   use std::sync::atomic::Ordering;
 

cli/ops.rs

@@ -1952,12 +1952,14 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
+    tokio_util::init(move || {
       let fetch_result =
         op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
       match fetch_result {
         Ok(_) => assert!(true),
         Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
       }
+    })
   }
 
   #[test]
@@ -1986,12 +1988,14 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
+    tokio_util::init(move || {
       let fetch_result =
         op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
       match fetch_result {
         Ok(_) => assert!(true),
         Err(e) => assert_eq!(e.to_string(), permission_denied().to_string()),
       }
+    })
   }
 
   #[test]
@@ -2019,12 +2023,15 @@ mod tests {
     msg::finish_base_buffer(builder, base);
     let data = builder.finished_data();
     let final_msg = msg::get_root_as_base(&data);
+
+    tokio_util::init(move || {
       let fetch_result =
         op_fetch_module_meta_data(&cli, &final_msg, deno_buf::empty()).wait();
       match fetch_result {
         Ok(_) => assert!(true),
         Err(e) => assert!(e.to_string() != permission_denied().to_string()),
       }
+    })
   }
 }
 */