denoland-deno/cli/specifier_handler.rs

// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.

use crate::ast::Location;
use crate::disk_cache::DiskCache;
use crate::file_fetcher::FileFetcher;
use crate::proc_state::ProcState;

use deno_ast::MediaType;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures::future;
use deno_core::futures::Future;
use deno_core::futures::FutureExt;
use deno_core::serde::Deserialize;
use deno_core::serde::Serialize;
use deno_core::serde_json;
use deno_core::ModuleSpecifier;
use deno_runtime::permissions::Permissions;
use log::debug;
use std::collections::HashMap;
use std::fmt;
use std::path::PathBuf;
use std::pin::Pin;
use std::sync::Arc;

pub type DependencyMap = HashMap<String, Dependency>;
type FetchFutureOutput = Result<CachedModule, (ModuleSpecifier, AnyError)>;
pub type FetchFuture = Pin<Box<dyn Future<Output = FetchFutureOutput> + Send>>;

/// A group of errors that represent errors that can occur with an
/// an implementation of `SpecifierHandler`.
#[derive(Debug, Clone, Eq, PartialEq)]
pub enum HandlerError {
  /// A fetch error, where we have a location associated with it.
  FetchErrorWithLocation(String, Location),
}

impl fmt::Display for HandlerError {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    match self {
      HandlerError::FetchErrorWithLocation(ref err, ref location) => {
        write!(f, "{}\n    at {}", err, location)
      }
    }
  }
}

impl std::error::Error for HandlerError {}

#[derive(Debug, Clone)]
pub struct CachedModule {
  pub is_remote: bool,
  pub maybe_dependencies: Option<DependencyMap>,
  pub maybe_emit: Option<Emit>,
  pub maybe_emit_path: Option<(PathBuf, Option<PathBuf>)>,
  pub maybe_types: Option<String>,
  pub maybe_version: Option<String>,
  pub media_type: MediaType,
  pub requested_specifier: ModuleSpecifier,
  pub source: Arc<String>,
  pub source_path: PathBuf,
  pub specifier: ModuleSpecifier,
}

impl Default for CachedModule {
  fn default() -> Self {
    let specifier = deno_core::resolve_url("file:///example.js").unwrap();
    CachedModule {
      is_remote: false,
      maybe_dependencies: None,
      maybe_emit: None,
      maybe_emit_path: None,
      maybe_types: None,
      maybe_version: None,
      media_type: MediaType::Unknown,
      requested_specifier: specifier.clone(),
      source: Arc::new(String::default()),
      source_path: PathBuf::new(),
      specifier,
    }
  }
}

/// An enum to own the a specific emit.
///
/// Currently there is only one type of emit that is cacheable, but this has
/// been added to future proof the ability for the specifier handler
/// implementations to be able to handle other types of emits, like form a
/// runtime API which might have a different configuration.
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum Emit {
  /// Code that was emitted for use by the CLI
  Cli((String, Option<String>)),
}

impl Default for Emit {
  fn default() -> Self {
    Emit::Cli(("".to_string(), None))
  }
}

#[derive(Debug, Clone)]
pub struct Dependency {
  /// Flags if the dependency is a dynamic import or not.
  pub is_dynamic: bool,
  /// The location in the source code where the dependency statement occurred.
  pub location: Location,
  /// The module specifier that resolves to the runtime code dependency for the
  /// module.
  pub maybe_code: Option<ModuleSpecifier>,
  /// The module specifier that resolves to the type only dependency for the
  /// module.
  pub maybe_type: Option<ModuleSpecifier>,
}

impl Dependency {
  pub fn new(location: Location) -> Self {
    Dependency {
      is_dynamic: false,
      location,
      maybe_code: None,
      maybe_type: None,
    }
  }
}

pub trait SpecifierHandler: Sync + Send {
  /// Instructs the handler to fetch a specifier or retrieve its value from the
  /// cache.
  fn fetch(
    &mut self,
    specifier: ModuleSpecifier,
    maybe_location: Option<Location>,
    is_dynamic: bool,
  ) -> FetchFuture;

  /// Get the optional build info from the cache for a given module specifier.
  /// Because build infos are only associated with the "root" modules, they are
  /// not expected to be cached for each module, but are "lazily" checked when
  /// a root module is identified.  The `emit_type` also indicates what form
  /// of the module the build info is valid for.
  fn get_tsbuildinfo(
    &self,
    specifier: &ModuleSpecifier,
  ) -> Result<Option<String>, AnyError>;

  /// Set the emit for the module specifier.
  fn set_cache(
    &mut self,
    specifier: &ModuleSpecifier,
    emit: &Emit,
  ) -> Result<(), AnyError>;

  /// When parsed out of a JavaScript module source, the triple slash reference
  /// to the types should be stored in the cache.
  fn set_types(
    &mut self,
    specifier: &ModuleSpecifier,
    types: String,
  ) -> Result<(), AnyError>;

  /// Set the build info for a module specifier, also providing the cache type.
  fn set_tsbuildinfo(
    &mut self,
    specifier: &ModuleSpecifier,
    tsbuildinfo: String,
  ) -> Result<(), AnyError>;

  /// Set the graph dependencies for a given module specifier.
  fn set_deps(
    &mut self,
    specifier: &ModuleSpecifier,
    dependencies: DependencyMap,
  ) -> Result<(), AnyError>;

  /// Set the version of the source for a given module, which is used to help
  /// determine if a module needs to be re-emitted.
  fn set_version(
    &mut self,
    specifier: &ModuleSpecifier,
    version: String,
  ) -> Result<(), AnyError>;
}

impl fmt::Debug for dyn SpecifierHandler {
  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
    write!(f, "SpecifierHandler {{ }}")
  }
}

/// A representation of meta data for a compiled file.
///
/// *Note* this is currently just a copy of what is located in `tsc.rs` but will
/// be refactored to be able to store dependencies and type information in the
/// future.
#[derive(Deserialize, Serialize)]
pub struct CompiledFileMetadata {
  pub version_hash: String,
}

impl CompiledFileMetadata {
  pub fn from_bytes(bytes: &[u8]) -> Result<Self, AnyError> {
    let metadata_string = std::str::from_utf8(bytes)?;
    serde_json::from_str::<Self>(metadata_string).map_err(|e| e.into())
  }

  pub fn to_json_string(&self) -> Result<String, AnyError> {
    serde_json::to_string(self).map_err(|e| e.into())
  }
}

/// An implementation of the `SpecifierHandler` trait that integrates with the
/// existing `file_fetcher` interface, which will eventually be refactored to
/// align it more to the `SpecifierHandler` trait.
pub struct FetchHandler {
  /// An instance of disk where generated (emitted) files are stored.
  disk_cache: DiskCache,
  /// The set permissions which are used for root modules (static imports).
  root_permissions: Permissions,
  /// The set of permissions which are used for dynamic imports.
  dynamic_permissions: Permissions,
  /// A clone of the `ps` file fetcher.
  file_fetcher: FileFetcher,
}

impl FetchHandler {
  pub fn new(
    ps: &ProcState,
    root_permissions: Permissions,
    dynamic_permissions: Permissions,
  ) -> Result<Self, AnyError> {
    let disk_cache = ps.dir.gen_cache.clone();
    let file_fetcher = ps.file_fetcher.clone();

    Ok(FetchHandler {
      disk_cache,
      root_permissions,
      dynamic_permissions,
      file_fetcher,
    })
  }
}

impl SpecifierHandler for FetchHandler {
  fn fetch(
    &mut self,
    requested_specifier: ModuleSpecifier,
    maybe_location: Option<Location>,
    is_dynamic: bool,
  ) -> FetchFuture {
    // When the module graph fetches dynamic modules, the set of dynamic
    // permissions need to be applied.  Other static imports have all
    // permissions.
    let mut permissions = if is_dynamic {
      self.dynamic_permissions.clone()
    } else {
      self.root_permissions.clone()
    };
    let file_fetcher = self.file_fetcher.clone();
    let disk_cache = self.disk_cache.clone();

    async move {
      let source_file = file_fetcher
        .fetch(&requested_specifier, &mut permissions)
        .await
        .map_err(|err| {
          let err = if let Some(e) = err.downcast_ref::<std::io::Error>() {
            if e.kind() == std::io::ErrorKind::NotFound {
              let message = if let Some(location) = &maybe_location {
                format!(
                  "Cannot resolve module \"{}\" from \"{}\".",
                  requested_specifier, location.specifier
                )
              } else {
                format!("Cannot resolve module \"{}\".", requested_specifier)
              };
              custom_error("NotFound", message)
            } else {
              err
            }
          } else {
            err
          };
          if let Some(location) = maybe_location {
            // Injected modules (like test and eval) come with locations, but
            // they are confusing to the user to print out the location because
            // they cannot actually get to the source code that is quoted, as
            // it only exists in the runtime memory of Deno.
            if !location.specifier.contains("$deno$") {
              (
                requested_specifier.clone(),
                HandlerError::FetchErrorWithLocation(err.to_string(), location)
                  .into(),
              )
            } else {
              (requested_specifier.clone(), err)
            }
          } else {
            (requested_specifier.clone(), err)
          }
        })?;
      let url = &source_file.specifier;
      let is_remote = !(url.scheme() == "file"
        || url.scheme() == "data"
        || url.scheme() == "blob");
      let filename = disk_cache.get_cache_filename_with_extension(url, "meta");
      let maybe_version = if let Some(filename) = filename {
        if let Ok(bytes) = disk_cache.get(&filename) {
          if let Ok(compiled_file_metadata) =
            CompiledFileMetadata::from_bytes(&bytes)
          {
            Some(compiled_file_metadata.version_hash)
          } else {
            None
          }
        } else {
          None
        }
      } else {
        None
      };

      let mut maybe_map_path = None;
      let map_path =
        disk_cache.get_cache_filename_with_extension(url, "js.map");
      let maybe_map = if let Some(map_path) = map_path {
        if let Ok(map) = disk_cache.get(&map_path) {
          maybe_map_path = Some(disk_cache.location.join(map_path));
          Some(String::from_utf8(map).unwrap())
        } else {
          None
        }
      } else {
        None
      };
      let mut maybe_emit = None;
      let mut maybe_emit_path = None;
      let emit_path = disk_cache.get_cache_filename_with_extension(url, "js");
      if let Some(emit_path) = emit_path {
        if let Ok(code) = disk_cache.get(&emit_path) {
          maybe_emit =
            Some(Emit::Cli((String::from_utf8(code).unwrap(), maybe_map)));
          maybe_emit_path =
            Some((disk_cache.location.join(emit_path), maybe_map_path));
        }
      };

      Ok(CachedModule {
        is_remote,
        maybe_dependencies: None,
        maybe_emit,
        maybe_emit_path,
        maybe_types: source_file.maybe_types,
        maybe_version,
        media_type: source_file.media_type,
        requested_specifier,
        source: source_file.source,
        source_path: source_file.local,
        specifier: source_file.specifier,
      })
    }
    .boxed()
  }

  fn get_tsbuildinfo(
    &self,
    specifier: &ModuleSpecifier,
  ) -> Result<Option<String>, AnyError> {
    let filename = self
      .disk_cache
      .get_cache_filename_with_extension(specifier, "buildinfo");
    if let Some(filename) = filename {
      if let Ok(tsbuildinfo) = self.disk_cache.get(&filename) {
        Ok(Some(String::from_utf8(tsbuildinfo)?))
      } else {
        Ok(None)
      }
    } else {
      Ok(None)
    }
  }

  fn set_tsbuildinfo(
    &mut self,
    specifier: &ModuleSpecifier,
    tsbuildinfo: String,
  ) -> Result<(), AnyError> {
    let filename = self
      .disk_cache
      .get_cache_filename_with_extension(specifier, "buildinfo")
      .unwrap();
    debug!("set_tsbuildinfo - filename {:?}", filename);
    self
      .disk_cache
      .set(&filename, tsbuildinfo.as_bytes())
      .map_err(|e| e.into())
  }

  fn set_cache(
    &mut self,
    specifier: &ModuleSpecifier,
    emit: &Emit,
  ) -> Result<(), AnyError> {
    match emit {
      Emit::Cli((code, maybe_map)) => {
        let filename = self
          .disk_cache
          .get_cache_filename_with_extension(specifier, "js")
          .unwrap();
        self.disk_cache.set(&filename, code.as_bytes())?;

        if let Some(map) = maybe_map {
          let filename = self
            .disk_cache
            .get_cache_filename_with_extension(specifier, "js.map")
            .unwrap();
          self.disk_cache.set(&filename, map.as_bytes())?;
        }
      }
    };

    Ok(())
  }

  fn set_deps(
    &mut self,
    _specifier: &ModuleSpecifier,
    _dependencies: DependencyMap,
  ) -> Result<(), AnyError> {
    // file_fetcher doesn't have the concept of caching dependencies
    Ok(())
  }

  fn set_types(
    &mut self,
    _specifier: &ModuleSpecifier,
    _types: String,
  ) -> Result<(), AnyError> {
    // file_fetcher doesn't have the concept of caching of the types
    Ok(())
  }

  fn set_version(
    &mut self,
    specifier: &ModuleSpecifier,
    version_hash: String,
  ) -> Result<(), AnyError> {
    let compiled_file_metadata = CompiledFileMetadata { version_hash };
    let filename = self
      .disk_cache
      .get_cache_filename_with_extension(specifier, "meta")
      .unwrap();

    self
      .disk_cache
      .set(
        &filename,
        compiled_file_metadata.to_json_string()?.as_bytes(),
      )
      .map_err(|e| e.into())
  }
}

pub struct MemoryHandler {
  sources: HashMap<String, Arc<String>>,
}

impl MemoryHandler {
  pub fn new(sources: HashMap<String, Arc<String>>) -> Self {
    Self { sources }
  }
}

impl SpecifierHandler for MemoryHandler {
  fn fetch(
    &mut self,
    specifier: ModuleSpecifier,
    _maybe_referrer: Option<Location>,
    _is_dynamic: bool,
  ) -> FetchFuture {
    let mut specifier_text = specifier.to_string();
    if !self.sources.contains_key(&specifier_text) {
      specifier_text = specifier_text.replace("file:///", "/");
      if !self.sources.contains_key(&specifier_text) {
        // Convert `C:/a/path/file.ts` to `/a/path/file.ts`
        specifier_text = specifier_text[3..].to_string()
      }
    }
    let result = if let Some(source) = self.sources.get(&specifier_text) {
      let media_type = MediaType::from(&specifier);
      let is_remote = specifier.scheme() != "file";

      Ok(CachedModule {
        source: source.clone(),
        requested_specifier: specifier.clone(),
        specifier,
        media_type,
        is_remote,
        ..Default::default()
      })
    } else {
      Err((
        specifier.clone(),
        custom_error(
          "NotFound",
          format!("Unable to find specifier in sources: {}", specifier),
        ),
      ))
    };

    Box::pin(future::ready(result))
  }

  fn get_tsbuildinfo(
    &self,
    _specifier: &ModuleSpecifier,
  ) -> Result<Option<String>, AnyError> {
    Ok(None)
  }

  fn set_cache(
    &mut self,
    _specifier: &ModuleSpecifier,
    _emit: &Emit,
  ) -> Result<(), AnyError> {
    Ok(())
  }

  fn set_types(
    &mut self,
    _specifier: &ModuleSpecifier,
    _types: String,
  ) -> Result<(), AnyError> {
    Ok(())
  }

  fn set_tsbuildinfo(
    &mut self,
    _specifier: &ModuleSpecifier,
    _tsbuildinfo: String,
  ) -> Result<(), AnyError> {
    Ok(())
  }

  fn set_deps(
    &mut self,
    _specifier: &ModuleSpecifier,
    _dependencies: DependencyMap,
  ) -> Result<(), AnyError> {
    Ok(())
  }

  fn set_version(
    &mut self,
    _specifier: &ModuleSpecifier,
    _version: String,
  ) -> Result<(), AnyError> {
    Ok(())
  }
}

#[cfg(test)]
pub mod tests {
  use super::*;
  use crate::deno_dir::DenoDir;
  use crate::file_fetcher::CacheSetting;
  use crate::http_cache::HttpCache;
  use deno_core::resolve_url_or_path;
  use deno_runtime::deno_web::BlobStore;
  use tempfile::TempDir;

  macro_rules! map (
    { $($key:expr => $value:expr),+ } => {
      {
        let mut m = ::std::collections::HashMap::new();
        $(
          m.insert($key, $value);
        )+
        m
      }
    };
  );

  fn setup() -> (TempDir, FetchHandler) {
    let temp_dir = TempDir::new().expect("could not setup");
    let deno_dir = DenoDir::new(Some(temp_dir.path().to_path_buf()))
      .expect("could not setup");

    let file_fetcher = FileFetcher::new(
      HttpCache::new(&temp_dir.path().to_path_buf().join("deps")),
      CacheSetting::Use,
      true,
      None,
      BlobStore::default(),
      None,
    )
    .expect("could not setup");
    let disk_cache = deno_dir.gen_cache;

    let fetch_handler = FetchHandler {
      disk_cache,
      root_permissions: Permissions::allow_all(),
      dynamic_permissions: Permissions::default(),
      file_fetcher,
    };

    (temp_dir, fetch_handler)
  }

  #[tokio::test]
  async fn test_fetch_handler_fetch() {
    let _http_server_guard = test_util::http_server();
    let (_, mut file_fetcher) = setup();
    let specifier =
      resolve_url_or_path("http://localhost:4545/subdir/mod2.ts").unwrap();
    let cached_module: CachedModule = file_fetcher
      .fetch(specifier.clone(), None, false)
      .await
      .unwrap();
    assert!(cached_module.maybe_emit.is_none());
    assert!(cached_module.maybe_dependencies.is_none());
    assert_eq!(cached_module.media_type, MediaType::TypeScript);
    assert_eq!(
      cached_module.source.as_str(),
      "export { printHello } from \"./print_hello.ts\";\n"
    );
    assert_eq!(cached_module.specifier, specifier);
  }

  #[tokio::test]
  async fn test_fetch_handler_set_cache() {
    let _http_server_guard = test_util::http_server();
    let (_, mut file_fetcher) = setup();
    let specifier =
      resolve_url_or_path("http://localhost:4545/subdir/mod2.ts").unwrap();
    let cached_module: CachedModule = file_fetcher
      .fetch(specifier.clone(), None, false)
      .await
      .unwrap();
    assert!(cached_module.maybe_emit.is_none());
    let code = String::from("some code");
    file_fetcher
      .set_cache(&specifier, &Emit::Cli((code, None)))
      .expect("could not set cache");
    let cached_module: CachedModule = file_fetcher
      .fetch(specifier.clone(), None, false)
      .await
      .unwrap();
    assert_eq!(
      cached_module.maybe_emit,
      Some(Emit::Cli(("some code".to_string(), None)))
    );
  }

  #[tokio::test]
  async fn test_fetch_handler_is_remote() {
    let _http_server_guard = test_util::http_server();
    let (_, mut file_fetcher) = setup();
    let specifier =
      resolve_url_or_path("http://localhost:4545/subdir/mod2.ts").unwrap();
    let cached_module: CachedModule =
      file_fetcher.fetch(specifier, None, false).await.unwrap();
    assert!(cached_module.is_remote);
    let specifier = resolve_url_or_path(
      test_util::testdata_path()
        .join("subdir/mod1.ts")
        .as_os_str()
        .to_str()
        .unwrap(),
    )
    .unwrap();
    let cached_module: CachedModule =
      file_fetcher.fetch(specifier, None, false).await.unwrap();
    assert!(!cached_module.is_remote);
  }

  #[tokio::test]
  async fn test_memory_handler_fetch() {
    let a_src = r#"
      import * as b from "./b.ts";
      console.log(b);
    "#;
    let b_src = r#"
      export const b = "b";
    "#;
    let c_src = r#"
      export const c = "c";
    "#;
    let d_src = r#"
      export const d: string;
    "#;
    let sources = map!(
      "/a.ts" => a_src,
      "/b.ts" => b_src,
      "https://deno.land/x/c.js" => c_src,
      "https://deno.land/x/d.d.ts" => d_src
    );
    let sources: HashMap<String, Arc<String>> = sources
      .iter()
      .map(|(k, v)| (k.to_string(), Arc::new(v.to_string())))
      .collect();
    let mut handler = MemoryHandler::new(sources);
    let specifier = resolve_url_or_path("file:///a.ts").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), a_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::TypeScript);
    assert!(!actual.is_remote);

    let specifier = resolve_url_or_path("file:///b.ts").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), b_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::TypeScript);
    assert!(!actual.is_remote);

    let specifier = resolve_url_or_path("https://deno.land/x/c.js").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), c_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::JavaScript);
    assert!(actual.is_remote);

    let specifier = resolve_url_or_path("https://deno.land/x/d.d.ts").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), d_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::Dts);
    assert!(actual.is_remote);

    let specifier =
      resolve_url_or_path("https://deno.land/x/missing.ts").unwrap();
    handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect_err("should have errored");

    let specifier = resolve_url_or_path("/a.ts").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), a_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::TypeScript);
    assert!(!actual.is_remote);

    let specifier = resolve_url_or_path("file:///C:/a.ts").unwrap();
    let actual: CachedModule = handler
      .fetch(specifier.clone(), None, false)
      .await
      .expect("could not fetch module");
    assert_eq!(actual.source.as_str(), a_src);
    assert_eq!(actual.requested_specifier, specifier);
    assert_eq!(actual.specifier, specifier);
    assert_eq!(actual.media_type, MediaType::TypeScript);
    assert!(!actual.is_remote);
  }
}