denoland-deno/cli/module_graph.rs

// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.

use crate::file_fetcher::map_file_extension;
use crate::file_fetcher::SourceFile;
use crate::file_fetcher::SourceFileFetcher;
use crate::import_map::ImportMap;
use crate::msg::MediaType;
use crate::op_error::OpError;
use crate::permissions::Permissions;
use crate::swc_util::analyze_dependencies_and_references;
use crate::swc_util::TsReferenceKind;
use crate::tsc::get_available_libs;
use deno_core::ErrBox;
use deno_core::ModuleSpecifier;
use futures::stream::FuturesUnordered;
use futures::stream::StreamExt;
use futures::Future;
use futures::FutureExt;
use serde::Serialize;
use serde::Serializer;
use std::collections::HashMap;
use std::collections::HashSet;
use std::hash::BuildHasher;
use std::path::PathBuf;
use std::pin::Pin;

fn serialize_module_specifier<S>(
  spec: &ModuleSpecifier,
  s: S,
) -> Result<S::Ok, S::Error>
where
  S: Serializer,
{
  s.serialize_str(&spec.to_string())
}

fn serialize_option_module_specifier<S>(
  maybe_spec: &Option<ModuleSpecifier>,
  s: S,
) -> Result<S::Ok, S::Error>
where
  S: Serializer,
{
  if let Some(spec) = maybe_spec {
    serialize_module_specifier(spec, s)
  } else {
    s.serialize_none()
  }
}

#[derive(Debug, Serialize)]
pub struct ModuleGraph(HashMap<String, ModuleGraphFile>);

#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ImportDescriptor {
  specifier: String,
  #[serde(serialize_with = "serialize_module_specifier")]
  resolved_specifier: ModuleSpecifier,
  // These two fields are for support of @deno-types directive
  // directly prepending import statement
  type_directive: Option<String>,
  #[serde(serialize_with = "serialize_option_module_specifier")]
  resolved_type_directive: Option<ModuleSpecifier>,
}

#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ReferenceDescriptor {
  specifier: String,
  #[serde(serialize_with = "serialize_module_specifier")]
  resolved_specifier: ModuleSpecifier,
}

#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ModuleGraphFile {
  pub specifier: String,
  pub url: String,
  pub redirect: Option<String>,
  pub filename: String,
  pub imports: Vec<ImportDescriptor>,
  pub referenced_files: Vec<ReferenceDescriptor>,
  pub lib_directives: Vec<ReferenceDescriptor>,
  pub types_directives: Vec<ReferenceDescriptor>,
  pub type_headers: Vec<ReferenceDescriptor>,
  pub media_type: i32,
  pub source_code: String,
}

type SourceFileFuture =
  Pin<Box<dyn Future<Output = Result<(ModuleSpecifier, SourceFile), ErrBox>>>>;

pub struct ModuleGraphLoader {
  permissions: Permissions,
  file_fetcher: SourceFileFetcher,
  maybe_import_map: Option<ImportMap>,
  pending_downloads: FuturesUnordered<SourceFileFuture>,
  has_downloaded: HashSet<ModuleSpecifier>,
  pub graph: ModuleGraph,
  is_dyn_import: bool,
  analyze_dynamic_imports: bool,
}

impl ModuleGraphLoader {
  pub fn new(
    file_fetcher: SourceFileFetcher,
    maybe_import_map: Option<ImportMap>,
    permissions: Permissions,
    is_dyn_import: bool,
    analyze_dynamic_imports: bool,
  ) -> Self {
    Self {
      file_fetcher,
      permissions,
      maybe_import_map,
      pending_downloads: FuturesUnordered::new(),
      has_downloaded: HashSet::new(),
      graph: ModuleGraph(HashMap::new()),
      is_dyn_import,
      analyze_dynamic_imports,
    }
  }

  /// This method is used to add specified module and all of its
  /// dependencies to the graph.
  ///
  /// It resolves when all dependent modules have been fetched and analyzed.
  ///
  /// This method can be called multiple times.
  pub async fn add_to_graph(
    &mut self,
    specifier: &ModuleSpecifier,
  ) -> Result<(), ErrBox> {
    self.download_module(specifier.clone(), None)?;

    loop {
      let (specifier, source_file) =
        self.pending_downloads.next().await.unwrap()?;
      self.visit_module(&specifier, source_file)?;
      if self.pending_downloads.is_empty() {
        break;
      }
    }

    Ok(())
  }

  /// This method is used to create a graph from in-memory files stored in
  /// a hash map. Useful for creating module graph for code received from
  /// the runtime.
  pub fn build_local_graph<S: BuildHasher>(
    &mut self,
    _root_name: &str,
    source_map: &HashMap<String, String, S>,
  ) -> Result<(), ErrBox> {
    for (spec, source_code) in source_map.iter() {
      self.visit_memory_module(spec.to_string(), source_code.to_string())?;
    }

    Ok(())
  }

  /// Consumes the loader and returns created graph.
  pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
    self.graph.0
  }

  fn visit_memory_module(
    &mut self,
    specifier: String,
    source_code: String,
  ) -> Result<(), ErrBox> {
    let mut imports = vec![];
    let mut referenced_files = vec![];
    let mut lib_directives = vec![];
    let mut types_directives = vec![];

    // FIXME(bartlomieju):
    // The resolveModules op only handles fully qualified URLs for referrer.
    // However we will have cases where referrer is "/foo.ts". We add this dummy
    // prefix "memory://" in order to use resolution logic.
    let module_specifier =
      if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) {
        spec
      } else {
        ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))?
      };

    let (import_descs, ref_descs) = analyze_dependencies_and_references(
      &specifier,
      &source_code,
      self.analyze_dynamic_imports,
    )?;

    for import_desc in import_descs {
      let maybe_resolved =
        if let Some(import_map) = self.maybe_import_map.as_ref() {
          import_map
            .resolve(&import_desc.specifier, &module_specifier.to_string())?
        } else {
          None
        };

      let resolved_specifier = if let Some(resolved) = maybe_resolved {
        resolved
      } else {
        ModuleSpecifier::resolve_import(
          &import_desc.specifier,
          &module_specifier.to_string(),
        )?
      };

      let resolved_type_directive =
        if let Some(types_specifier) = import_desc.deno_types.as_ref() {
          Some(ModuleSpecifier::resolve_import(
            &types_specifier,
            &module_specifier.to_string(),
          )?)
        } else {
          None
        };

      let import_descriptor = ImportDescriptor {
        specifier: import_desc.specifier.to_string(),
        resolved_specifier,
        type_directive: import_desc.deno_types,
        resolved_type_directive,
      };

      imports.push(import_descriptor);
    }

    let available_libs = get_available_libs();

    for ref_desc in ref_descs {
      if available_libs.contains(&ref_desc.specifier) {
        continue;
      }

      let resolved_specifier = ModuleSpecifier::resolve_import(
        &ref_desc.specifier,
        &module_specifier.to_string(),
      )?;

      let reference_descriptor = ReferenceDescriptor {
        specifier: ref_desc.specifier.to_string(),
        resolved_specifier,
      };

      match ref_desc.kind {
        TsReferenceKind::Lib => {
          lib_directives.push(reference_descriptor);
        }
        TsReferenceKind::Types => {
          types_directives.push(reference_descriptor);
        }
        TsReferenceKind::Path => {
          referenced_files.push(reference_descriptor);
        }
      }
    }

    self.graph.0.insert(
      module_specifier.to_string(),
      ModuleGraphFile {
        specifier: specifier.to_string(),
        url: specifier.to_string(),
        redirect: None,
        media_type: map_file_extension(&PathBuf::from(specifier.clone()))
          as i32,
        filename: specifier,
        source_code,
        imports,
        referenced_files,
        lib_directives,
        types_directives,
        type_headers: vec![],
      },
    );
    Ok(())
  }

  // TODO(bartlomieju): decorate errors with import location in the source code
  // https://github.com/denoland/deno/issues/5080
  fn download_module(
    &mut self,
    module_specifier: ModuleSpecifier,
    maybe_referrer: Option<ModuleSpecifier>,
  ) -> Result<(), ErrBox> {
    if self.has_downloaded.contains(&module_specifier) {
      return Ok(());
    }

    // Disallow http:// imports from modules loaded over https://
    if let Some(referrer) = maybe_referrer.as_ref() {
      if let "https" = referrer.as_url().scheme() {
        if let "http" = module_specifier.as_url().scheme() {
          let e = OpError::permission_denied(
            "Modules loaded over https:// are not allowed to import modules over http://".to_string()
          );
          return Err(e.into());
        };
      };
    };

    if !self.is_dyn_import {
      // Verify that remote file doesn't try to statically import local file.
      if let Some(referrer) = maybe_referrer.as_ref() {
        let referrer_url = referrer.as_url();
        match referrer_url.scheme() {
          "http" | "https" => {
            let specifier_url = module_specifier.as_url();
            match specifier_url.scheme() {
              "http" | "https" => {}
              _ => {
                let e = OpError::permission_denied(
                  "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string()
                );
                return Err(e.into());
              }
            }
          }
          _ => {}
        }
      }
    }

    self.has_downloaded.insert(module_specifier.clone());
    let spec = module_specifier;
    let file_fetcher = self.file_fetcher.clone();
    let perms = self.permissions.clone();

    let load_future = async move {
      let spec_ = spec.clone();
      let source_file = file_fetcher
        .fetch_source_file(&spec_, maybe_referrer, perms)
        .await?;
      Ok((spec_.clone(), source_file))
    }
    .boxed_local();

    self.pending_downloads.push(load_future);
    Ok(())
  }

  fn visit_module(
    &mut self,
    module_specifier: &ModuleSpecifier,
    source_file: SourceFile,
  ) -> Result<(), ErrBox> {
    let mut imports = vec![];
    let mut referenced_files = vec![];
    let mut lib_directives = vec![];
    let mut types_directives = vec![];
    let mut type_headers = vec![];

    // IMPORTANT: source_file.url might be different than requested
    // module_specifier because of HTTP redirects. In such
    // situation we add an "empty" ModuleGraphFile with 'redirect'
    // field set that will be later used in TS worker when building
    // map of available source file. It will perform substitution
    // for proper URL point to redirect target.
    if module_specifier.as_url() != &source_file.url {
      // TODO(bartlomieju): refactor, this is a band-aid
      self.graph.0.insert(
        module_specifier.to_string(),
        ModuleGraphFile {
          specifier: module_specifier.to_string(),
          url: module_specifier.to_string(),
          redirect: Some(source_file.url.to_string()),
          filename: source_file.filename.to_str().unwrap().to_string(),
          media_type: source_file.media_type as i32,
          source_code: "".to_string(),
          imports: vec![],
          referenced_files: vec![],
          lib_directives: vec![],
          types_directives: vec![],
          type_headers: vec![],
        },
      );
    }

    let module_specifier = ModuleSpecifier::from(source_file.url.clone());
    let source_code = String::from_utf8(source_file.source_code)?;

    if source_file.media_type == MediaType::JavaScript
      || source_file.media_type == MediaType::TypeScript
    {
      if let Some(types_specifier) = source_file.types_header {
        let type_header = ReferenceDescriptor {
          specifier: types_specifier.to_string(),
          resolved_specifier: ModuleSpecifier::resolve_import(
            &types_specifier,
            &module_specifier.to_string(),
          )?,
        };
        self.download_module(
          type_header.resolved_specifier.clone(),
          Some(module_specifier.clone()),
        )?;
        type_headers.push(type_header);
      }

      let (import_descs, ref_descs) = analyze_dependencies_and_references(
        &module_specifier.to_string(),
        &source_code,
        self.analyze_dynamic_imports,
      )?;

      for import_desc in import_descs {
        let maybe_resolved =
          if let Some(import_map) = self.maybe_import_map.as_ref() {
            import_map
              .resolve(&import_desc.specifier, &module_specifier.to_string())?
          } else {
            None
          };

        let resolved_specifier = if let Some(resolved) = maybe_resolved {
          resolved
        } else {
          ModuleSpecifier::resolve_import(
            &import_desc.specifier,
            &module_specifier.to_string(),
          )?
        };

        let resolved_type_directive =
          if let Some(types_specifier) = import_desc.deno_types.as_ref() {
            Some(ModuleSpecifier::resolve_import(
              &types_specifier,
              &module_specifier.to_string(),
            )?)
          } else {
            None
          };

        let import_descriptor = ImportDescriptor {
          specifier: import_desc.specifier.to_string(),
          resolved_specifier,
          type_directive: import_desc.deno_types,
          resolved_type_directive,
        };

        self.download_module(
          import_descriptor.resolved_specifier.clone(),
          Some(module_specifier.clone()),
        )?;

        if let Some(type_dir_url) =
          import_descriptor.resolved_type_directive.as_ref()
        {
          self.download_module(
            type_dir_url.clone(),
            Some(module_specifier.clone()),
          )?;
        }

        imports.push(import_descriptor);
      }

      let available_libs = get_available_libs();

      for ref_desc in ref_descs {
        if available_libs.contains(&ref_desc.specifier) {
          continue;
        }

        let resolved_specifier = ModuleSpecifier::resolve_import(
          &ref_desc.specifier,
          &module_specifier.to_string(),
        )?;

        let reference_descriptor = ReferenceDescriptor {
          specifier: ref_desc.specifier.to_string(),
          resolved_specifier,
        };

        self.download_module(
          reference_descriptor.resolved_specifier.clone(),
          Some(module_specifier.clone()),
        )?;

        match ref_desc.kind {
          TsReferenceKind::Lib => {
            lib_directives.push(reference_descriptor);
          }
          TsReferenceKind::Types => {
            types_directives.push(reference_descriptor);
          }
          TsReferenceKind::Path => {
            referenced_files.push(reference_descriptor);
          }
        }
      }
    }

    self.graph.0.insert(
      module_specifier.to_string(),
      ModuleGraphFile {
        specifier: module_specifier.to_string(),
        url: module_specifier.to_string(),
        redirect: None,
        filename: source_file.filename.to_str().unwrap().to_string(),
        media_type: source_file.media_type as i32,
        source_code,
        imports,
        referenced_files,
        lib_directives,
        types_directives,
        type_headers,
      },
    );
    Ok(())
  }
}

#[cfg(test)]
mod tests {
  use super::*;
  use crate::GlobalState;

  async fn build_graph(
    module_specifier: &ModuleSpecifier,
  ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
    let global_state = GlobalState::new(Default::default()).unwrap();
    let mut graph_loader = ModuleGraphLoader::new(
      global_state.file_fetcher.clone(),
      None,
      Permissions::allow_all(),
      false,
      false,
    );
    graph_loader.add_to_graph(&module_specifier).await?;
    Ok(graph_loader.get_graph())
  }

  // TODO(bartlomieju): this test is flaky, because it's using 019_media_types
  // file, reenable once Python server is replaced with Rust one.
  #[ignore]
  #[tokio::test]
  async fn source_graph_fetch() {
    let http_server_guard = crate::test_util::http_server();

    let module_specifier = ModuleSpecifier::resolve_url_or_path(
      "http://localhost:4545/cli/tests/019_media_types.ts",
    )
    .unwrap();
    let graph = build_graph(&module_specifier)
      .await
      .expect("Failed to build graph");

    let a = graph
      .get("http://localhost:4545/cli/tests/019_media_types.ts")
      .unwrap();

    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js"
    ));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts"
    ));
    assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts"));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts"
    ));
    assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js"));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js"
    ));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js"
    ));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts"
    ));

    assert_eq!(
      serde_json::to_value(&a.imports).unwrap(),
      json!([
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
        {
          "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
      ])
    );
    drop(http_server_guard);
  }

  #[tokio::test]
  async fn source_graph_type_references() {
    let http_server_guard = crate::test_util::http_server();

    let module_specifier = ModuleSpecifier::resolve_url_or_path(
      "http://localhost:4545/cli/tests/type_definitions.ts",
    )
    .unwrap();

    let graph = build_graph(&module_specifier)
      .await
      .expect("Failed to build graph");

    eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap());

    let a = graph
      .get("http://localhost:4545/cli/tests/type_definitions.ts")
      .unwrap();
    assert_eq!(
      serde_json::to_value(&a.imports).unwrap(),
      json!([
        {
          "specifier": "./type_definitions/foo.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js",
          "typeDirective": "./type_definitions/foo.d.ts",
          "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
        },
        {
          "specifier": "./type_definitions/fizz.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js",
          "typeDirective": "./type_definitions/fizz.d.ts",
          "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
        },
        {
          "specifier": "./type_definitions/qat.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        },
      ])
    );
    assert!(graph
      .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js"));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/type_definitions/foo.d.ts"
    ));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/type_definitions/fizz.js"
    ));
    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts"
    ));
    assert!(graph
      .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts"));

    drop(http_server_guard);
  }

  #[tokio::test]
  async fn source_graph_type_references2() {
    let http_server_guard = crate::test_util::http_server();

    let module_specifier = ModuleSpecifier::resolve_url_or_path(
      "http://localhost:4545/cli/tests/type_directives_02.ts",
    )
    .unwrap();

    let graph = build_graph(&module_specifier)
      .await
      .expect("Failed to build graph");

    eprintln!("{:#?}", serde_json::to_value(&graph).unwrap());

    let a = graph
      .get("http://localhost:4545/cli/tests/type_directives_02.ts")
      .unwrap();
    assert_eq!(
      serde_json::to_value(&a.imports).unwrap(),
      json!([
        {
          "specifier": "./subdir/type_reference.js",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        }
      ])
    );

    assert!(graph.contains_key(
      "http://localhost:4545/cli/tests/subdir/type_reference.d.ts"
    ));

    let b = graph
      .get("http://localhost:4545/cli/tests/subdir/type_reference.js")
      .unwrap();
    assert_eq!(
      serde_json::to_value(&b.types_directives).unwrap(),
      json!([
        {
          "specifier": "./type_reference.d.ts",
          "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts",
        }
      ])
    );
    drop(http_server_guard);
  }

  #[tokio::test]
  async fn source_graph_type_references3() {
    let http_server_guard = crate::test_util::http_server();

    let module_specifier = ModuleSpecifier::resolve_url_or_path(
      "http://localhost:4545/cli/tests/type_directives_01.ts",
    )
    .unwrap();

    let graph = build_graph(&module_specifier)
      .await
      .expect("Failed to build graph");

    let ts = graph
      .get("http://localhost:4545/cli/tests/type_directives_01.ts")
      .unwrap();
    assert_eq!(
      serde_json::to_value(&ts.imports).unwrap(),
      json!([
        {
          "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
          "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js",
          "typeDirective": null,
          "resolvedTypeDirective": null,
        }
      ])
    );

    let headers = graph
      .get("http://127.0.0.1:4545/xTypeScriptTypes.js")
      .unwrap();
    assert_eq!(
      serde_json::to_value(&headers.type_headers).unwrap(),
      json!([
        {
          "specifier": "./xTypeScriptTypes.d.ts",
          "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts"
        }
      ])
    );
    drop(http_server_guard);
  }
}