// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::file_fetcher::map_file_extension; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::op_error::OpError; use crate::permissions::Permissions; use crate::swc_util::analyze_dependencies_and_references; use crate::swc_util::TsReferenceKind; use crate::tsc::get_available_libs; use deno_core::ErrBox; use deno_core::ModuleSpecifier; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; use futures::Future; use futures::FutureExt; use serde::Serialize; use serde::Serializer; use std::collections::HashMap; use std::collections::HashSet; use std::hash::BuildHasher; use std::path::PathBuf; use std::pin::Pin; fn serialize_module_specifier( spec: &ModuleSpecifier, s: S, ) -> Result where S: Serializer, { s.serialize_str(&spec.to_string()) } fn serialize_option_module_specifier( maybe_spec: &Option, s: S, ) -> Result where S: Serializer, { if let Some(spec) = maybe_spec { serialize_module_specifier(spec, s) } else { s.serialize_none() } } const SUPPORTED_MEDIA_TYPES: [MediaType; 4] = [ MediaType::JavaScript, MediaType::TypeScript, MediaType::JSX, MediaType::TSX, ]; #[derive(Debug, Serialize)] pub struct ModuleGraph(HashMap); #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct ImportDescriptor { specifier: String, #[serde(serialize_with = "serialize_module_specifier")] resolved_specifier: ModuleSpecifier, // These two fields are for support of @deno-types directive // directly prepending import statement type_directive: Option, #[serde(serialize_with = "serialize_option_module_specifier")] resolved_type_directive: Option, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct ReferenceDescriptor { specifier: String, #[serde(serialize_with = "serialize_module_specifier")] resolved_specifier: ModuleSpecifier, } #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] pub struct ModuleGraphFile { pub specifier: String, pub url: String, pub redirect: Option, pub filename: String, pub imports: Vec, pub referenced_files: Vec, pub lib_directives: Vec, pub types_directives: Vec, pub type_headers: Vec, pub media_type: i32, pub source_code: String, } type SourceFileFuture = Pin>>>; pub struct ModuleGraphLoader { permissions: Permissions, file_fetcher: SourceFileFetcher, maybe_import_map: Option, pending_downloads: FuturesUnordered, has_downloaded: HashSet, pub graph: ModuleGraph, is_dyn_import: bool, analyze_dynamic_imports: bool, } impl ModuleGraphLoader { pub fn new( file_fetcher: SourceFileFetcher, maybe_import_map: Option, permissions: Permissions, is_dyn_import: bool, analyze_dynamic_imports: bool, ) -> Self { Self { file_fetcher, permissions, maybe_import_map, pending_downloads: FuturesUnordered::new(), has_downloaded: HashSet::new(), graph: ModuleGraph(HashMap::new()), is_dyn_import, analyze_dynamic_imports, } } /// This method is used to add specified module and all of its /// dependencies to the graph. /// /// It resolves when all dependent modules have been fetched and analyzed. /// /// This method can be called multiple times. pub async fn add_to_graph( &mut self, specifier: &ModuleSpecifier, ) -> Result<(), ErrBox> { self.download_module(specifier.clone(), None)?; loop { let (specifier, source_file) = self.pending_downloads.next().await.unwrap()?; self.visit_module(&specifier, source_file)?; if self.pending_downloads.is_empty() { break; } } Ok(()) } /// This method is used to create a graph from in-memory files stored in /// a hash map. Useful for creating module graph for code received from /// the runtime. pub fn build_local_graph( &mut self, _root_name: &str, source_map: &HashMap, ) -> Result<(), ErrBox> { for (spec, source_code) in source_map.iter() { self.visit_memory_module(spec.to_string(), source_code.to_string())?; } Ok(()) } /// Consumes the loader and returns created graph. pub fn get_graph(self) -> HashMap { self.graph.0 } fn visit_memory_module( &mut self, specifier: String, source_code: String, ) -> Result<(), ErrBox> { let mut imports = vec![]; let mut referenced_files = vec![]; let mut lib_directives = vec![]; let mut types_directives = vec![]; // FIXME(bartlomieju): // The resolveModules op only handles fully qualified URLs for referrer. // However we will have cases where referrer is "/foo.ts". We add this dummy // prefix "memory://" in order to use resolution logic. let module_specifier = if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) { spec } else { ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))? }; let (import_descs, ref_descs) = analyze_dependencies_and_references( &specifier, map_file_extension(&PathBuf::from(&specifier)), &source_code, self.analyze_dynamic_imports, )?; for import_desc in import_descs { let maybe_resolved = if let Some(import_map) = self.maybe_import_map.as_ref() { import_map .resolve(&import_desc.specifier, &module_specifier.to_string())? } else { None }; let resolved_specifier = if let Some(resolved) = maybe_resolved { resolved } else { ModuleSpecifier::resolve_import( &import_desc.specifier, &module_specifier.to_string(), )? }; let resolved_type_directive = if let Some(types_specifier) = import_desc.deno_types.as_ref() { Some(ModuleSpecifier::resolve_import( &types_specifier, &module_specifier.to_string(), )?) } else { None }; let import_descriptor = ImportDescriptor { specifier: import_desc.specifier.to_string(), resolved_specifier, type_directive: import_desc.deno_types, resolved_type_directive, }; imports.push(import_descriptor); } let available_libs = get_available_libs(); for ref_desc in ref_descs { if available_libs.contains(&ref_desc.specifier) { continue; } let resolved_specifier = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), )?; let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), resolved_specifier, }; match ref_desc.kind { TsReferenceKind::Lib => { lib_directives.push(reference_descriptor); } TsReferenceKind::Types => { types_directives.push(reference_descriptor); } TsReferenceKind::Path => { referenced_files.push(reference_descriptor); } } } self.graph.0.insert( module_specifier.to_string(), ModuleGraphFile { specifier: specifier.to_string(), url: specifier.to_string(), redirect: None, media_type: map_file_extension(&PathBuf::from(specifier.clone())) as i32, filename: specifier, source_code, imports, referenced_files, lib_directives, types_directives, type_headers: vec![], }, ); Ok(()) } // TODO(bartlomieju): decorate errors with import location in the source code // https://github.com/denoland/deno/issues/5080 fn download_module( &mut self, module_specifier: ModuleSpecifier, maybe_referrer: Option, ) -> Result<(), ErrBox> { if self.has_downloaded.contains(&module_specifier) { return Ok(()); } // Disallow http:// imports from modules loaded over https:// if let Some(referrer) = maybe_referrer.as_ref() { if let "https" = referrer.as_url().scheme() { if let "http" = module_specifier.as_url().scheme() { let e = OpError::permission_denied( "Modules loaded over https:// are not allowed to import modules over http://".to_string() ); return Err(e.into()); }; }; }; if !self.is_dyn_import { // Verify that remote file doesn't try to statically import local file. if let Some(referrer) = maybe_referrer.as_ref() { let referrer_url = referrer.as_url(); match referrer_url.scheme() { "http" | "https" => { let specifier_url = module_specifier.as_url(); match specifier_url.scheme() { "http" | "https" => {} _ => { let e = OpError::permission_denied( "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string() ); return Err(e.into()); } } } _ => {} } } } self.has_downloaded.insert(module_specifier.clone()); let spec = module_specifier; let file_fetcher = self.file_fetcher.clone(); let perms = self.permissions.clone(); let load_future = async move { let spec_ = spec.clone(); let source_file = file_fetcher .fetch_source_file(&spec_, maybe_referrer, perms) .await?; Ok((spec_.clone(), source_file)) } .boxed_local(); self.pending_downloads.push(load_future); Ok(()) } fn visit_module( &mut self, module_specifier: &ModuleSpecifier, source_file: SourceFile, ) -> Result<(), ErrBox> { let mut imports = vec![]; let mut referenced_files = vec![]; let mut lib_directives = vec![]; let mut types_directives = vec![]; let mut type_headers = vec![]; // IMPORTANT: source_file.url might be different than requested // module_specifier because of HTTP redirects. In such // situation we add an "empty" ModuleGraphFile with 'redirect' // field set that will be later used in TS worker when building // map of available source file. It will perform substitution // for proper URL point to redirect target. if module_specifier.as_url() != &source_file.url { // TODO(bartlomieju): refactor, this is a band-aid self.graph.0.insert( module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), url: module_specifier.to_string(), redirect: Some(source_file.url.to_string()), filename: source_file.filename.to_str().unwrap().to_string(), media_type: source_file.media_type as i32, source_code: "".to_string(), imports: vec![], referenced_files: vec![], lib_directives: vec![], types_directives: vec![], type_headers: vec![], }, ); } let module_specifier = ModuleSpecifier::from(source_file.url.clone()); let source_code = String::from_utf8(source_file.source_code)?; if SUPPORTED_MEDIA_TYPES.contains(&source_file.media_type) { if let Some(types_specifier) = source_file.types_header { let type_header = ReferenceDescriptor { specifier: types_specifier.to_string(), resolved_specifier: ModuleSpecifier::resolve_import( &types_specifier, &module_specifier.to_string(), )?, }; self.download_module( type_header.resolved_specifier.clone(), Some(module_specifier.clone()), )?; type_headers.push(type_header); } let (import_descs, ref_descs) = analyze_dependencies_and_references( &module_specifier.to_string(), source_file.media_type, &source_code, self.analyze_dynamic_imports, )?; for import_desc in import_descs { let maybe_resolved = if let Some(import_map) = self.maybe_import_map.as_ref() { import_map .resolve(&import_desc.specifier, &module_specifier.to_string())? } else { None }; let resolved_specifier = if let Some(resolved) = maybe_resolved { resolved } else { ModuleSpecifier::resolve_import( &import_desc.specifier, &module_specifier.to_string(), )? }; let resolved_type_directive = if let Some(types_specifier) = import_desc.deno_types.as_ref() { Some(ModuleSpecifier::resolve_import( &types_specifier, &module_specifier.to_string(), )?) } else { None }; let import_descriptor = ImportDescriptor { specifier: import_desc.specifier.to_string(), resolved_specifier, type_directive: import_desc.deno_types, resolved_type_directive, }; self.download_module( import_descriptor.resolved_specifier.clone(), Some(module_specifier.clone()), )?; if let Some(type_dir_url) = import_descriptor.resolved_type_directive.as_ref() { self.download_module( type_dir_url.clone(), Some(module_specifier.clone()), )?; } imports.push(import_descriptor); } let available_libs = get_available_libs(); for ref_desc in ref_descs { if available_libs.contains(&ref_desc.specifier) { continue; } let resolved_specifier = ModuleSpecifier::resolve_import( &ref_desc.specifier, &module_specifier.to_string(), )?; let reference_descriptor = ReferenceDescriptor { specifier: ref_desc.specifier.to_string(), resolved_specifier, }; self.download_module( reference_descriptor.resolved_specifier.clone(), Some(module_specifier.clone()), )?; match ref_desc.kind { TsReferenceKind::Lib => { lib_directives.push(reference_descriptor); } TsReferenceKind::Types => { types_directives.push(reference_descriptor); } TsReferenceKind::Path => { referenced_files.push(reference_descriptor); } } } } self.graph.0.insert( module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), url: module_specifier.to_string(), redirect: None, filename: source_file.filename.to_str().unwrap().to_string(), media_type: source_file.media_type as i32, source_code, imports, referenced_files, lib_directives, types_directives, type_headers, }, ); Ok(()) } } #[cfg(test)] mod tests { use super::*; use crate::GlobalState; async fn build_graph( module_specifier: &ModuleSpecifier, ) -> Result, ErrBox> { let global_state = GlobalState::new(Default::default()).unwrap(); let mut graph_loader = ModuleGraphLoader::new( global_state.file_fetcher.clone(), None, Permissions::allow_all(), false, false, ); graph_loader.add_to_graph(&module_specifier).await?; Ok(graph_loader.get_graph()) } // TODO(bartlomieju): this test is flaky, because it's using 019_media_types // file, reenable once Python server is replaced with Rust one. #[ignore] #[tokio::test] async fn source_graph_fetch() { let http_server_guard = crate::test_util::http_server(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/019_media_types.ts", ) .unwrap(); let graph = build_graph(&module_specifier) .await .expect("Failed to build graph"); let a = graph .get("http://localhost:4545/cli/tests/019_media_types.ts") .unwrap(); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js" )); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts" )); assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts")); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts" )); assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js")); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js" )); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js" )); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts" )); assert_eq!( serde_json::to_value(&a.imports).unwrap(), json!([ { "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", "typeDirective": null, "resolvedTypeDirective": null, }, { "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", "typeDirective": null, "resolvedTypeDirective": null, }, ]) ); drop(http_server_guard); } #[tokio::test] async fn source_graph_type_references() { let http_server_guard = crate::test_util::http_server(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_definitions.ts", ) .unwrap(); let graph = build_graph(&module_specifier) .await .expect("Failed to build graph"); eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); let a = graph .get("http://localhost:4545/cli/tests/type_definitions.ts") .unwrap(); assert_eq!( serde_json::to_value(&a.imports).unwrap(), json!([ { "specifier": "./type_definitions/foo.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", "typeDirective": "./type_definitions/foo.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" }, { "specifier": "./type_definitions/fizz.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", "typeDirective": "./type_definitions/fizz.d.ts", "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" }, { "specifier": "./type_definitions/qat.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", "typeDirective": null, "resolvedTypeDirective": null, }, ]) ); assert!(graph .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js")); assert!(graph.contains_key( "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" )); assert!(graph.contains_key( "http://localhost:4545/cli/tests/type_definitions/fizz.js" )); assert!(graph.contains_key( "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" )); assert!(graph .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts")); drop(http_server_guard); } #[tokio::test] async fn source_graph_type_references2() { let http_server_guard = crate::test_util::http_server(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_directives_02.ts", ) .unwrap(); let graph = build_graph(&module_specifier) .await .expect("Failed to build graph"); eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); let a = graph .get("http://localhost:4545/cli/tests/type_directives_02.ts") .unwrap(); assert_eq!( serde_json::to_value(&a.imports).unwrap(), json!([ { "specifier": "./subdir/type_reference.js", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", "typeDirective": null, "resolvedTypeDirective": null, } ]) ); assert!(graph.contains_key( "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" )); let b = graph .get("http://localhost:4545/cli/tests/subdir/type_reference.js") .unwrap(); assert_eq!( serde_json::to_value(&b.types_directives).unwrap(), json!([ { "specifier": "./type_reference.d.ts", "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", } ]) ); drop(http_server_guard); } #[tokio::test] async fn source_graph_type_references3() { let http_server_guard = crate::test_util::http_server(); let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/type_directives_01.ts", ) .unwrap(); let graph = build_graph(&module_specifier) .await .expect("Failed to build graph"); let ts = graph .get("http://localhost:4545/cli/tests/type_directives_01.ts") .unwrap(); assert_eq!( serde_json::to_value(&ts.imports).unwrap(), json!([ { "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", "typeDirective": null, "resolvedTypeDirective": null, } ]) ); let headers = graph .get("http://127.0.0.1:4545/xTypeScriptTypes.js") .unwrap(); assert_eq!( serde_json::to_value(&headers.type_headers).unwrap(), json!([ { "specifier": "./xTypeScriptTypes.d.ts", "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" } ]) ); drop(http_server_guard); } #[tokio::test] async fn source_graph_different_langs() { let http_server_guard = crate::test_util::http_server(); // ModuleGraphLoader was mistakenly parsing this file as TSX // https://github.com/denoland/deno/issues/5867 let module_specifier = ModuleSpecifier::resolve_url_or_path( "http://localhost:4545/cli/tests/ts_with_generic.ts", ) .unwrap(); build_graph(&module_specifier) .await .expect("Failed to build graph"); drop(http_server_guard); } }