feat: support loading import map from URL (#9519)

This commit adds support for loading import maps from URLs,
both remote and local.

This feature is supported in CLI flag as well as in runtime
compiler API.

cli/import_map.rs

@@ -1,6 +1,5 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_core::serde_json::Map;
 use deno_core::serde_json::Value;
@@ -10,8 +9,6 @@ use indexmap::IndexMap;
 use std::cmp::Ordering;
 use std::error::Error;
 use std::fmt;
-use std::fs;
-use std::io;
 
 #[derive(Debug)]
 pub struct ImportMapError {
@@ -49,30 +46,6 @@ pub struct ImportMap {
 }
 
 impl ImportMap {
-  pub fn load(file_path: &str) -> Result<Self, AnyError> {
-    let file_url = ModuleSpecifier::resolve_url_or_path(file_path)?.to_string();
-    let resolved_path = std::env::current_dir().unwrap().join(file_path);
-    debug!(
-      "Attempt to load import map: {}",
-      resolved_path.to_str().unwrap()
-    );
-
-    // Load the contents of import map
-    let json_string = fs::read_to_string(&resolved_path).map_err(|err| {
-      io::Error::new(
-        io::ErrorKind::InvalidInput,
-        format!(
-          "Error retrieving import map file at \"{}\": {}",
-          resolved_path.to_str().unwrap(),
-          err.to_string()
-        )
-        .as_str(),
-      )
-    })?;
-    // The URL of the import map is the base URL for its values.
-    ImportMap::from_json(&file_url, &json_string).map_err(AnyError::from)
-  }
-
   pub fn from_json(
     base_url: &str,
     json_string: &str,
@@ -476,12 +449,6 @@ mod tests {
   use super::*;
   use deno_core::serde_json::json;
 
-  #[test]
-  fn load_nonexistent() {
-    let file_path = "nonexistent_import_map.json";
-    assert!(ImportMap::load(file_path).is_err());
-  }
-
   #[test]
   fn from_json_1() {
     let base_url = "https://deno.land";

cli/lsp/sources.rs

@@ -31,7 +31,7 @@ pub async fn cache(
   specifier: &ModuleSpecifier,
   maybe_import_map: &Option<ImportMap>,
 ) -> Result<(), AnyError> {
-  let program_state = Arc::new(ProgramState::new(Default::default())?);
+  let program_state = Arc::new(ProgramState::build(Default::default()).await?);
   let handler = Arc::new(Mutex::new(FetchHandler::new(
     &program_state,
     Permissions::allow_all(),

cli/main.rs

@@ -313,7 +313,7 @@ async fn compile_command(
     tools::standalone::compile_to_runtime_flags(flags.clone(), args)?;
 
   let module_specifier = ModuleSpecifier::resolve_url_or_path(&source_file)?;
-  let program_state = ProgramState::new(flags.clone())?;
+  let program_state = ProgramState::build(flags.clone()).await?;
   let deno_dir = &program_state.dir;
 
   let output = output.or_else(|| {
@@ -367,7 +367,7 @@ async fn info_command(
   if json && !flags.unstable {
     exit_unstable("--json");
   }
-  let program_state = ProgramState::new(flags)?;
+  let program_state = ProgramState::build(flags).await?;
   if let Some(specifier) = maybe_specifier {
     let specifier = ModuleSpecifier::resolve_url_or_path(&specifier)?;
     let handler = Arc::new(Mutex::new(specifier_handler::FetchHandler::new(
@@ -409,7 +409,7 @@ async fn install_command(
   preload_flags.inspect = None;
   preload_flags.inspect_brk = None;
   let permissions = Permissions::from_options(&preload_flags.clone().into());
-  let program_state = ProgramState::new(preload_flags)?;
+  let program_state = ProgramState::build(preload_flags).await?;
   let main_module = ModuleSpecifier::resolve_url_or_path(&module_url)?;
   let mut worker =
     create_main_worker(&program_state, main_module.clone(), permissions);
@@ -450,7 +450,7 @@ async fn cache_command(
   } else {
     module_graph::TypeLib::DenoWindow
   };
-  let program_state = ProgramState::new(flags)?;
+  let program_state = ProgramState::build(flags).await?;
 
   for file in files {
     let specifier = ModuleSpecifier::resolve_url_or_path(&file)?;
@@ -478,7 +478,7 @@ async fn eval_command(
   let main_module =
     ModuleSpecifier::resolve_url_or_path("./$deno$eval.ts").unwrap();
   let permissions = Permissions::from_options(&flags.clone().into());
-  let program_state = ProgramState::new(flags)?;
+  let program_state = ProgramState::build(flags).await?;
   let mut worker =
     create_main_worker(&program_state, main_module.clone(), permissions);
   let main_module_url = main_module.as_url().to_owned();
@@ -596,7 +596,7 @@ async fn bundle_command(
         ModuleSpecifier::resolve_url_or_path(&source_file1)?;
 
       debug!(">>>>> bundle START");
-      let program_state = ProgramState::new(flags.clone())?;
+      let program_state = ProgramState::build(flags.clone()).await?;
 
       info!(
         "{} {}",
@@ -742,7 +742,7 @@ async fn doc_command(
   maybe_filter: Option<String>,
   private: bool,
 ) -> Result<(), AnyError> {
-  let program_state = ProgramState::new(flags.clone())?;
+  let program_state = ProgramState::build(flags.clone()).await?;
   let source_file = source_file.unwrap_or_else(|| "--builtin".to_string());
 
   let loader = Box::new(DocLoader {
@@ -822,7 +822,7 @@ async fn run_repl(flags: Flags) -> Result<(), AnyError> {
   let main_module =
     ModuleSpecifier::resolve_url_or_path("./$deno$repl.ts").unwrap();
   let permissions = Permissions::from_options(&flags.clone().into());
-  let program_state = ProgramState::new(flags)?;
+  let program_state = ProgramState::build(flags).await?;
   let mut worker =
     create_main_worker(&program_state, main_module.clone(), permissions);
   worker.run_event_loop().await?;
@@ -831,7 +831,7 @@ async fn run_repl(flags: Flags) -> Result<(), AnyError> {
 }
 
 async fn run_from_stdin(flags: Flags) -> Result<(), AnyError> {
-  let program_state = ProgramState::new(flags.clone())?;
+  let program_state = ProgramState::build(flags.clone()).await?;
   let permissions = Permissions::from_options(&flags.clone().into());
   let main_module =
     ModuleSpecifier::resolve_url_or_path("./$deno$stdin.ts").unwrap();
@@ -871,7 +871,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> {
     let flags = flags.clone();
     async move {
       let main_module = ModuleSpecifier::resolve_url_or_path(&script1)?;
-      let program_state = ProgramState::new(flags)?;
+      let program_state = ProgramState::build(flags).await?;
       let handler = Arc::new(Mutex::new(FetchHandler::new(
         &program_state,
         Permissions::allow_all(),
@@ -916,7 +916,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> {
     let permissions = Permissions::from_options(&flags.clone().into());
     async move {
       let main_module = main_module.clone();
-      let program_state = ProgramState::new(flags)?;
+      let program_state = ProgramState::build(flags).await?;
       let mut worker =
         create_main_worker(&program_state, main_module.clone(), permissions);
       debug!("main_module {}", main_module);
@@ -948,7 +948,7 @@ async fn run_command(flags: Flags, script: String) -> Result<(), AnyError> {
   }
 
   let main_module = ModuleSpecifier::resolve_url_or_path(&script)?;
-  let program_state = ProgramState::new(flags.clone())?;
+  let program_state = ProgramState::build(flags.clone()).await?;
   let permissions = Permissions::from_options(&flags.clone().into());
   let mut worker =
     create_main_worker(&program_state, main_module.clone(), permissions);
@@ -989,7 +989,7 @@ async fn test_command(
   allow_none: bool,
   filter: Option<String>,
 ) -> Result<(), AnyError> {
-  let program_state = ProgramState::new(flags.clone())?;
+  let program_state = ProgramState::build(flags.clone()).await?;
   let permissions = Permissions::from_options(&flags.clone().into());
   let cwd = std::env::current_dir().expect("No current directory");
   let include = include.unwrap_or_else(|| vec![".".to_string()]);

cli/ops/runtime_compiler.rs

@@ -75,19 +75,22 @@ async fn op_emit(
       is_dynamic = true;
       Arc::new(Mutex::new(FetchHandler::new(
         &program_state,
-        runtime_permissions,
+        runtime_permissions.clone(),
       )?))
     };
   let maybe_import_map = if let Some(import_map_str) = args.import_map_path {
     let import_map_specifier =
-      ModuleSpecifier::resolve_url_or_path(&import_map_str).context(
+      ModuleSpecifier::resolve_url_or_path(&import_map_str)
-        format!("Bad file path (\"{}\") for import map.", import_map_str),
+        .context(format!("Bad URL (\"{}\") for import map.", import_map_str))?;
-      )?;
     let import_map_url = import_map_specifier.as_url();
     let import_map = if let Some(value) = args.import_map {
       ImportMap::from_json(&import_map_url.to_string(), &value.to_string())?
     } else {
-      ImportMap::load(&import_map_str)?
+      let file = program_state
+        .file_fetcher
+        .fetch(&import_map_specifier, &runtime_permissions)
+        .await?;
+      ImportMap::from_json(import_map_specifier.as_str(), &file.source)?
     };
     Some(import_map)
   } else if args.import_map.is_some() {

cli/program_state.rs

@@ -56,7 +56,7 @@ pub struct ProgramState {
 }
 
 impl ProgramState {
-  pub fn new(flags: flags::Flags) -> Result<Arc<Self>, AnyError> {
+  pub async fn build(flags: flags::Flags) -> Result<Arc<Self>, AnyError> {
     let custom_root = env::var("DENO_DIR").map(String::into).ok();
     let dir = deno_dir::DenoDir::new(custom_root)?;
     let deps_cache_location = dir.root.join("deps");
@@ -94,11 +94,20 @@ impl ProgramState {
     let maybe_import_map: Option<ImportMap> =
       match flags.import_map_path.as_ref() {
         None => None,
-        Some(file_path) => {
+        Some(import_map_url) => {
           if !flags.unstable {
             exit_unstable("--import-map")
           }
-          Some(ImportMap::load(file_path)?)
+          let import_map_specifier =
+            ModuleSpecifier::resolve_url_or_path(&import_map_url).context(
+              format!("Bad URL (\"{}\") for import map.", import_map_url),
+            )?;
+          let file = file_fetcher
+            .fetch(&import_map_specifier, &Permissions::allow_all())
+            .await?;
+          let import_map =
+            ImportMap::from_json(import_map_specifier.as_str(), &file.source)?;
+          Some(import_map)
         }
       };
 
@@ -276,18 +285,6 @@ impl ProgramState {
       None
     }
   }
-
-  #[cfg(test)]
-  pub fn mock(
-    argv: Vec<String>,
-    maybe_flags: Option<flags::Flags>,
-  ) -> Arc<ProgramState> {
-    ProgramState::new(flags::Flags {
-      argv,
-      ..maybe_flags.unwrap_or_default()
-    })
-    .unwrap()
-  }
 }
 
 // TODO(@kitsonk) this is only temporary, but should be refactored to somewhere
@@ -346,14 +343,3 @@ fn source_map_from_code(code: String) -> Option<Vec<u8>> {
     None
   }
 }
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-
-  #[test]
-  fn thread_safe() {
-    fn f<S: Send + Sync>(_: S) {}
-    f(ProgramState::mock(vec![], None));
-  }
-}

cli/tests/033_import_map_remote.out

@@ -0,0 +1,5 @@
+Hello from remapped moment!
+Hello from remapped moment dir!
+Hello from remapped lodash!
+Hello from remapped lodash dir!
+Hello from remapped Vue!

cli/tests/import_maps/import_map_remote.json

@@ -0,0 +1,9 @@
+{
+  "imports": {
+    "moment": "./moment/moment.ts",
+    "moment/": "./moment/",
+    "lodash": "./lodash/lodash.ts",
+    "lodash/": "./lodash/",
+    "https://www.unpkg.com/vue/dist/vue.runtime.esm.js": "./vue.ts"
+  }
+}

cli/tests/import_maps/test_remote.ts

@@ -0,0 +1,5 @@
+import "moment";
+import "moment/other_file.ts";
+import "lodash";
+import "lodash/other_file.ts";
+import "https://www.unpkg.com/vue/dist/vue.runtime.esm.js";

cli/tests/integration_tests.rs

@@ -2475,6 +2475,13 @@ console.log("finish");
     output: "033_import_map.out",
   });
 
+  itest!(_033_import_map_remote {
+    args:
+      "run --quiet --reload --import-map=http://127.0.0.1:4545/cli/tests/import_maps/import_map_remote.json --unstable import_maps/test_remote.ts",
+    output: "033_import_map_remote.out",
+    http_server: true,
+  });
+
   itest!(_034_onload {
     args: "run --quiet --reload 034_onload/main.ts",
     output: "034_onload.out",