feat: Add --no-remote, rename --no-fetch to --cached-only (#3417)

cli/file_fetcher.rs

@@ -74,7 +74,8 @@ pub struct SourceFileFetcher {
   source_file_cache: SourceFileCache,
   cache_blacklist: Vec<String>,
   use_disk_cache: bool,
-  no_remote_fetch: bool,
+  no_remote: bool,
+  cached_only: bool,
 }
 
 impl SourceFileFetcher {
@@ -83,7 +84,8 @@ impl SourceFileFetcher {
     progress: Progress,
     use_disk_cache: bool,
     cache_blacklist: Vec<String>,
-    no_remote_fetch: bool,
+    no_remote: bool,
+    cached_only: bool,
   ) -> std::io::Result<Self> {
     let file_fetcher = Self {
       deps_cache,
@@ -91,7 +93,8 @@ impl SourceFileFetcher {
       source_file_cache: SourceFileCache::default(),
       cache_blacklist,
       use_disk_cache,
-      no_remote_fetch,
+      no_remote,
+      cached_only,
     };
 
     Ok(file_fetcher)
@@ -123,10 +126,10 @@ impl SourceFileFetcher {
 
     // If file is not in memory cache check if it can be found
     // in local cache - which effectively means trying to fetch
-    // using "--no-fetch" flag. We can safely block on this
+    // using "--cached-only" flag. We can safely block on this
     // future, because it doesn't do any asynchronous action
     // it that path.
-    let fut = self.get_source_file_async(specifier.as_url(), true, true);
+    let fut = self.get_source_file_async(specifier.as_url(), true, false, true);
 
     futures::executor::block_on(fut).ok()
   }
@@ -152,21 +155,31 @@ impl SourceFileFetcher {
       .get_source_file_async(
         &module_url,
         self.use_disk_cache,
-        self.no_remote_fetch,
+        self.no_remote,
+        self.cached_only,
       )
       .then(move |result| {
         let mut out = match result.map_err(|err| {
-          if err.kind() == ErrorKind::NotFound {
+          let err_kind = err.kind();
-            let msg = if let Some(referrer) = maybe_referrer {
+          let referrer_suffix = if let Some(referrer) = maybe_referrer {
-              format!(
+            format!(" from \"{}\"", referrer)
-                "Cannot resolve module \"{}\" from \"{}\"",
+          } else {
-                module_url.to_string(),
+            "".to_owned()
-                referrer
+          };
-              )
+          if err_kind == ErrorKind::NotFound {
-            } else {
+            let msg = format!(
-              format!("Cannot resolve module \"{}\"", module_url.to_string())
+              "Cannot resolve module \"{}\"{}",
-            };
+              module_url.to_string(),
+              referrer_suffix
+            );
             DenoError::new(ErrorKind::NotFound, msg).into()
+          } else if err_kind == ErrorKind::PermissionDenied {
+            let msg = format!(
+              "Cannot find module \"{}\"{} in cache, --cached-only is specified",
+              module_url.to_string(),
+              referrer_suffix
+            );
+            DenoError::new(ErrorKind::PermissionDenied, msg).into()
           } else {
             err
           }
@@ -196,13 +209,16 @@ impl SourceFileFetcher {
   ///
   /// If `use_disk_cache` is true then remote files are fetched from disk cache.
   ///
-  /// If `no_remote_fetch` is true then if remote file is not found it disk
+  /// If `no_remote` is true then this method will fail for remote files.
-  /// cache this method will fail.
+  ///
+  /// If `cached_only` is true then this method will fail for remote files
+  /// not already cached.
   fn get_source_file_async(
     self: &Self,
     module_url: &Url,
     use_disk_cache: bool,
-    no_remote_fetch: bool,
+    no_remote: bool,
+    cached_only: bool,
   ) -> impl Future<Output = Result<SourceFile, ErrBox>> {
     let url_scheme = module_url.scheme();
     let is_local_file = url_scheme == "file";
@@ -224,11 +240,25 @@ impl SourceFileFetcher {
       }
     }
 
+    // The file is remote, fail if `no_remote` is true.
+    if no_remote {
+      return Either::Left(futures::future::err(
+        std::io::Error::new(
+          std::io::ErrorKind::NotFound,
+          format!(
+            "Not allowed to get remote file '{}'",
+            module_url.to_string()
+          ),
+        )
+        .into(),
+      ));
+    }
+
     // Fetch remote file and cache on-disk for subsequent access
     Either::Right(self.fetch_remote_source_async(
       &module_url,
       use_disk_cache,
-      no_remote_fetch,
+      cached_only,
       10,
     ))
   }
@@ -327,7 +357,7 @@ impl SourceFileFetcher {
     self: &Self,
     module_url: &Url,
     use_disk_cache: bool,
-    no_remote_fetch: bool,
+    cached_only: bool,
     redirect_limit: i64,
   ) -> Pin<Box<SourceFileFuture>> {
     if redirect_limit < 0 {
@@ -352,11 +382,11 @@ impl SourceFileFetcher {
     }
 
     // If file wasn't found in cache check if we can fetch it
-    if no_remote_fetch {
+    if cached_only {
       // We can't fetch remote file - bail out
       return futures::future::err(
         std::io::Error::new(
-          std::io::ErrorKind::NotFound,
+          std::io::ErrorKind::PermissionDenied,
           format!(
             "cannot find remote file '{}' in cache",
             module_url.to_string()
@@ -391,7 +421,7 @@ impl SourceFileFetcher {
           Either::Left(dir.fetch_remote_source_async(
             &new_module_url,
             use_disk_cache,
-            no_remote_fetch,
+            cached_only,
             redirect_limit - 1,
           ))
         }
@@ -690,6 +720,7 @@ mod tests {
       true,
       vec![],
       false,
+      false,
     )
     .expect("setup fail")
   }
@@ -843,7 +874,7 @@ mod tests {
     let headers_file_name_3 = headers_file_name.clone();
 
     let fut = fetcher
-      .get_source_file_async(&module_url, true, false)
+      .get_source_file_async(&module_url, true, false, false)
       .then(move |result| {
         assert!(result.is_ok());
         let r = result.unwrap();
@@ -860,7 +891,7 @@ mod tests {
           &headers_file_name_1,
           "{ \"mime_type\": \"text/javascript\" }",
         );
-        fetcher_1.get_source_file_async(&module_url, true, false)
+        fetcher_1.get_source_file_async(&module_url, true, false, false)
       })
       .then(move |result2| {
         assert!(result2.is_ok());
@@ -886,7 +917,7 @@ mod tests {
           Some("application/json".to_owned()),
           None,
         );
-        fetcher_2.get_source_file_async(&module_url_1, true, false)
+        fetcher_2.get_source_file_async(&module_url_1, true, false, false)
       })
       .then(move |result3| {
         assert!(result3.is_ok());
@@ -905,7 +936,7 @@ mod tests {
         // let's create fresh instance of DenoDir (simulating another freshh Deno process)
         // and don't use cache
         let fetcher = setup_file_fetcher(temp_dir.path());
-        fetcher.get_source_file_async(&module_url_2, false, false)
+        fetcher.get_source_file_async(&module_url_2, false, false, false)
       })
       .then(move |result4| {
         assert!(result4.is_ok());
@@ -940,7 +971,7 @@ mod tests {
     );
 
     let fut = fetcher
-      .get_source_file_async(&module_url, true, false)
+      .get_source_file_async(&module_url, true, false, false)
       .then(move |result| {
         assert!(result.is_ok());
         let r = result.unwrap();
@@ -962,7 +993,7 @@ mod tests {
           Some("text/typescript".to_owned()),
           None,
         );
-        fetcher.get_source_file_async(&module_url, true, false)
+        fetcher.get_source_file_async(&module_url, true, false, false)
       })
       .then(move |result2| {
         assert!(result2.is_ok());
@@ -977,7 +1008,7 @@ mod tests {
         // let's create fresh instance of DenoDir (simulating another freshh Deno process)
         // and don't use cache
         let fetcher = setup_file_fetcher(temp_dir.path());
-        fetcher.get_source_file_async(&module_url_1, false, false)
+        fetcher.get_source_file_async(&module_url_1, false, false, false)
       })
       .then(move |result3| {
         assert!(result3.is_ok());
@@ -1077,7 +1108,7 @@ mod tests {
 
     // Test basic follow and headers recording
     let fut = fetcher
-      .get_source_file_async(&redirect_module_url, true, false)
+      .get_source_file_async(&redirect_module_url, true, false, false)
       .then(move |result| {
         assert!(result.is_ok());
         let mod_meta = result.unwrap();
@@ -1138,7 +1169,7 @@ mod tests {
 
     // Test double redirects and headers recording
     let fut = fetcher
-      .get_source_file_async(&double_redirect_url, true, false)
+      .get_source_file_async(&double_redirect_url, true, false, false)
       .then(move |result| {
         assert!(result.is_ok());
         let mod_meta = result.unwrap();
@@ -1196,7 +1227,7 @@ mod tests {
 
     // Test that redirect target is not downloaded twice for different redirect source.
     let fut = fetcher
-      .get_source_file_async(&double_redirect_url, true, false)
+      .get_source_file_async(&double_redirect_url, true, false, false)
       .then(move |result| {
         assert!(result.is_ok());
         let result = fs::File::open(&target_path);
@@ -1210,7 +1241,7 @@ mod tests {
         // shouldn't be downloaded again. It can be verified using source header file creation
         // timestamp (should be the same as after first `get_source_file`)
         fetcher
-          .get_source_file_async(&redirect_url, true, false)
+          .get_source_file_async(&redirect_url, true, false, false)
           .map(move |r| (r, file_modified))
       })
       .then(move |(result, file_modified)| {
@@ -1257,7 +1288,27 @@ mod tests {
   }
 
   #[test]
-  fn test_get_source_code_no_fetch() {
+  fn test_get_source_no_remote() {
+    let http_server_guard = crate::test_util::http_server();
+    let (_temp_dir, fetcher) = test_setup();
+    let module_url =
+      Url::parse("http://localhost:4545/tests/002_hello.ts").unwrap();
+    // Remote modules are not allowed
+    let fut = fetcher
+      .get_source_file_async(&module_url, true, true, false)
+      .then(move |result| {
+        assert!(result.is_err());
+        let err = result.err().unwrap();
+        assert_eq!(err.kind(), ErrorKind::NotFound);
+        futures::future::ok(())
+      });
+
+    tokio_util::run(fut);
+    drop(http_server_guard);
+  }
+
+  #[test]
+  fn test_get_source_cached_only() {
     let http_server_guard = crate::test_util::http_server();
     let (_temp_dir, fetcher) = test_setup();
     let fetcher_1 = fetcher.clone();
@@ -1266,21 +1317,21 @@ mod tests {
       Url::parse("http://localhost:4545/tests/002_hello.ts").unwrap();
     let module_url_1 = module_url.clone();
     let module_url_2 = module_url.clone();
-    // file hasn't been cached before and remote downloads are not allowed
+    // file hasn't been cached before
     let fut = fetcher
-      .get_source_file_async(&module_url, true, true)
+      .get_source_file_async(&module_url, true, false, true)
       .then(move |result| {
         assert!(result.is_err());
         let err = result.err().unwrap();
-        assert_eq!(err.kind(), ErrorKind::NotFound);
+        assert_eq!(err.kind(), ErrorKind::PermissionDenied);
 
         // download and cache file
-        fetcher_1.get_source_file_async(&module_url_1, true, false)
+        fetcher_1.get_source_file_async(&module_url_1, true, false, false)
       })
       .then(move |result| {
         assert!(result.is_ok());
-        // module is already cached, should be ok even with `no_remote_fetch`
+        // module is already cached, should be ok even with `cached_only`
-        fetcher_2.get_source_file_async(&module_url_2, true, true)
+        fetcher_2.get_source_file_async(&module_url_2, true, false, true)
       })
       .then(move |result| {
         assert!(result.is_ok());

cli/flags.rs

@@ -84,7 +84,8 @@ pub struct DenoFlags {
   pub allow_run: bool,
   pub allow_hrtime: bool,
   pub no_prompts: bool,
-  pub no_fetch: bool,
+  pub no_remote: bool,
+  pub cached_only: bool,
   pub seed: Option<u64>,
   pub v8_flags: Option<Vec<String>>,
   // Use tokio::runtime::current_thread
@@ -400,6 +401,7 @@ fn fetch_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
   lock_args_parse(flags, matches);
   importmap_arg_parse(flags, matches);
   config_arg_parse(flags, matches);
+  no_remote_arg_parse(flags, matches);
   if let Some(file) = matches.value_of("file") {
     flags.argv.push(file.into());
   }
@@ -422,6 +424,7 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
   importmap_arg_parse(flags, matches);
   config_arg_parse(flags, matches);
   v8_flags_arg_parse(flags, matches);
+  no_remote_arg_parse(flags, matches);
 
   if matches.is_present("allow-read") {
     if matches.value_of("allow-read").is_some() {
@@ -474,8 +477,8 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
     flags.allow_write = true;
     flags.allow_hrtime = true;
   }
-  if matches.is_present("no-fetch") {
+  if matches.is_present("cached-only") {
-    flags.no_fetch = true;
+    flags.cached_only = true;
   }
 
   if matches.is_present("current-thread") {
@@ -873,6 +876,7 @@ fn fetch_subcommand<'a, 'b>() -> App<'a, 'b> {
     .arg(lock_write_arg())
     .arg(importmap_arg())
     .arg(config_arg())
+    .arg(no_remote_arg())
     .arg(Arg::with_name("file").takes_value(true).required(true))
     .about("Fetch the dependencies")
     .long_about(
@@ -899,6 +903,7 @@ fn run_test_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
     .arg(config_arg())
     .arg(lock_arg())
     .arg(lock_write_arg())
+    .arg(no_remote_arg())
     .arg(v8_flags_arg())
     .arg(
       Arg::with_name("allow-read")
@@ -949,9 +954,9 @@ fn run_test_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
         .help("Allow all permissions"),
     )
     .arg(
-      Arg::with_name("no-fetch")
+      Arg::with_name("cached-only")
-        .long("no-fetch")
+        .long("cached-only")
-        .help("Do not download remote modules"),
+        .help("Require that remote dependencies are already cached"),
     )
     .arg(
       Arg::with_name("current-thread")
@@ -1150,6 +1155,18 @@ fn v8_flags_arg_parse(flags: &mut DenoFlags, matches: &ArgMatches) {
   }
 }
 
+fn no_remote_arg<'a, 'b>() -> Arg<'a, 'b> {
+  Arg::with_name("no-remote")
+    .long("no-remote")
+    .help("Do not resolve remote modules")
+}
+
+fn no_remote_arg_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
+  if matches.is_present("no-remote") {
+    flags.no_remote = true;
+  }
+}
+
 // TODO(ry) move this to utility module and add test.
 /// Strips fragment part of URL. Panics on bad URL.
 pub fn resolve_urls(urls: Vec<String>) -> Vec<String> {
@@ -2063,14 +2080,28 @@ mod tests {
   */
 
   #[test]
-  fn no_fetch() {
+  fn no_remote() {
-    let r = flags_from_vec_safe(svec!["deno", "--no-fetch", "script.ts"]);
+    let r = flags_from_vec_safe(svec!["deno", "--no-remote", "script.ts"]);
     assert_eq!(
       r.unwrap(),
       DenoFlags {
         subcommand: DenoSubcommand::Run,
         argv: svec!["deno", "script.ts"],
-        no_fetch: true,
+        no_remote: true,
+        ..DenoFlags::default()
+      }
+    );
+  }
+
+  #[test]
+  fn cached_only() {
+    let r = flags_from_vec_safe(svec!["deno", "--cached-only", "script.ts"]);
+    assert_eq!(
+      r.unwrap(),
+      DenoFlags {
+        subcommand: DenoSubcommand::Run,
+        argv: svec!["deno", "script.ts"],
+        cached_only: true,
         ..DenoFlags::default()
       }
     );

cli/global_state.rs

@@ -75,7 +75,8 @@ impl ThreadSafeGlobalState {
       progress.clone(),
       !flags.reload,
       flags.cache_blacklist.clone(),
-      flags.no_fetch,
+      flags.no_remote,
+      flags.cached_only,
     )?;
 
     let ts_compiler = TsCompiler::new(

cli/tests/035_cached_only_flag.out

@@ -0,0 +1 @@
+Cannot find module "http://127.0.0.1:4545/cli/tests/019_media_types.ts" in cache, --cached-only is specified

cli/tests/integration_tests.rs

@@ -296,10 +296,10 @@ itest!(_034_onload {
   output: "034_onload.out",
 });
 
-itest!(_035_no_fetch_flag {
+itest!(_035_cached_only_flag {
   args:
-    "--reload --no-fetch http://127.0.0.1:4545/cli/tests/019_media_types.ts",
+    "--reload --cached-only http://127.0.0.1:4545/cli/tests/019_media_types.ts",
-  output: "035_no_fetch_flag.out",
+  output: "035_cached_only_flag.out",
   exit_code: 1,
   check_stderr: true,
   http_server: true,
@@ -397,6 +397,15 @@ itest!(_051_wasm_import {
   http_server: true,
 });
 
+itest!(_052_no_remote_flag {
+  args:
+    "--reload --no-remote http://127.0.0.1:4545/cli/tests/019_media_types.ts",
+  output: "052_no_remote_flag.out",
+  exit_code: 1,
+  check_stderr: true,
+  http_server: true,
+});
+
 itest!(lock_check_ok {
   args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
   output: "003_relative_import.ts.out",