// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. use crate::colors; use crate::compat; use crate::config_file::ConfigFile; use crate::deno_dir; use crate::file_fetcher::CacheSetting; use crate::file_fetcher::FileFetcher; use crate::flags; use crate::http_cache; use crate::lockfile::Lockfile; use crate::module_graph::CheckOptions; use crate::module_graph::GraphBuilder; use crate::module_graph::TranspileOptions; use crate::module_graph::TypeLib; use crate::source_maps::SourceMapGetter; use crate::specifier_handler::FetchHandler; use crate::version; use deno_core::error::anyhow; use deno_core::error::get_custom_error_class; use deno_core::error::AnyError; use deno_core::error::Context; use deno_core::parking_lot::Mutex; use deno_core::resolve_url; use deno_core::url::Url; use deno_core::CompiledWasmModuleStore; use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use deno_core::SharedArrayBufferStore; use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::Permissions; use deno_tls::rustls::RootCertStore; use deno_tls::rustls_native_certs::load_native_certs; use deno_tls::webpki_roots::TLS_SERVER_ROOTS; use import_map::ImportMap; use log::debug; use log::info; use log::warn; use std::collections::HashMap; use std::collections::HashSet; use std::env; use std::fs::File; use std::io::BufReader; use std::ops::Deref; use std::sync::Arc; /// This structure represents state of single "deno" program. /// /// It is shared by all created workers (thus V8 isolates). #[derive(Clone)] pub struct ProcState(Arc); pub struct Inner { /// Flags parsed from `argv` contents. pub flags: flags::Flags, pub dir: deno_dir::DenoDir, pub coverage_dir: Option, pub file_fetcher: FileFetcher, pub modules: Arc>>>, pub lockfile: Option>>, pub maybe_config_file: Option, pub maybe_import_map: Option, pub maybe_inspector_server: Option>, pub root_cert_store: Option, pub blob_store: BlobStore, pub broadcast_channel: InMemoryBroadcastChannel, pub shared_array_buffer_store: SharedArrayBufferStore, pub compiled_wasm_module_store: CompiledWasmModuleStore, } impl Deref for ProcState { type Target = Arc; fn deref(&self) -> &Self::Target { &self.0 } } impl ProcState { pub async fn build(flags: flags::Flags) -> Result { let maybe_custom_root = flags .cache_path .clone() .or_else(|| env::var("DENO_DIR").map(String::into).ok()); let dir = deno_dir::DenoDir::new(maybe_custom_root)?; let deps_cache_location = dir.root.join("deps"); let http_cache = http_cache::HttpCache::new(&deps_cache_location); let mut root_cert_store = RootCertStore::empty(); let ca_stores: Vec = flags .ca_stores .clone() .or_else(|| { let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?; Some( env_ca_store .split(',') .map(|s| s.trim().to_string()) .filter(|s| !s.is_empty()) .collect(), ) }) .unwrap_or_else(|| vec!["mozilla".to_string()]); for store in ca_stores.iter() { match store.as_str() { "mozilla" => { root_cert_store.add_server_trust_anchors(&TLS_SERVER_ROOTS); } "system" => { let roots = load_native_certs() .expect("could not load platform certs") .roots; root_cert_store.roots.extend(roots); } _ => { return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store)); } } } let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok()); if let Some(ca_file) = ca_file { let certfile = File::open(&ca_file)?; let mut reader = BufReader::new(certfile); // This function does not return specific errors, if it fails give a generic message. if let Err(_err) = root_cert_store.add_pem_file(&mut reader) { return Err(anyhow!("Unable to add pem file to certificate store")); } } if let Some(insecure_allowlist) = flags.unsafely_ignore_certificate_errors.as_ref() { let domains = if insecure_allowlist.is_empty() { "for all hostnames".to_string() } else { format!("for: {}", insecure_allowlist.join(", ")) }; let msg = format!("DANGER: TLS certificate validation is disabled {}", domains); eprintln!("{}", colors::yellow(msg)); } let cache_usage = if flags.cached_only { CacheSetting::Only } else if !flags.cache_blocklist.is_empty() { CacheSetting::ReloadSome(flags.cache_blocklist.clone()) } else if flags.reload { CacheSetting::ReloadAll } else { CacheSetting::Use }; let blob_store = BlobStore::default(); let broadcast_channel = InMemoryBroadcastChannel::default(); let shared_array_buffer_store = SharedArrayBufferStore::default(); let compiled_wasm_module_store = CompiledWasmModuleStore::default(); let file_fetcher = FileFetcher::new( http_cache, cache_usage, !flags.no_remote, Some(root_cert_store.clone()), blob_store.clone(), flags.unsafely_ignore_certificate_errors.clone(), )?; let lockfile = if let Some(filename) = &flags.lock { let lockfile = Lockfile::new(filename.clone(), flags.lock_write)?; Some(Arc::new(Mutex::new(lockfile))) } else { None }; let maybe_config_file = if let Some(config_path) = flags.config_path.as_ref() { Some(ConfigFile::read(config_path)?) } else { None }; let mut maybe_import_map: Option = match flags.import_map_path.as_ref() { None => None, Some(import_map_url) => { let import_map_specifier = deno_core::resolve_url_or_path(import_map_url).context(format!( "Bad URL (\"{}\") for import map.", import_map_url ))?; let file = file_fetcher .fetch(&import_map_specifier, &mut Permissions::allow_all()) .await .context(format!( "Unable to load '{}' import map", import_map_specifier ))?; let import_map = ImportMap::from_json(import_map_specifier.as_str(), &file.source)?; Some(import_map) } }; if flags.compat { let mut import_map = match maybe_import_map { Some(import_map) => import_map, None => { // INFO: we're creating an empty import map, with its specifier pointing // to `CWD/node_import_map.json` to make sure the map still works as expected. let import_map_specifier = std::env::current_dir()?.join("node_import_map.json"); ImportMap::from_json(import_map_specifier.to_str().unwrap(), "{}") .unwrap() } }; let node_builtins = compat::get_mapped_node_builtins(); let diagnostics = import_map.update_imports(node_builtins)?; if !diagnostics.is_empty() { info!("Some Node built-ins were not added to the import map:"); for diagnostic in diagnostics { info!(" - {}", diagnostic); } info!("If you want to use Node built-ins provided by Deno remove listed specifiers from \"imports\" mapping in the import map file."); } maybe_import_map = Some(import_map); } let maybe_inspect_host = flags.inspect.or(flags.inspect_brk); let maybe_inspector_server = maybe_inspect_host.map(|host| { Arc::new(InspectorServer::new(host, version::get_user_agent())) }); let coverage_dir = flags .coverage_dir .clone() .or_else(|| env::var("DENO_UNSTABLE_COVERAGE_DIR").ok()); Ok(ProcState(Arc::new(Inner { dir, coverage_dir, flags, file_fetcher, modules: Default::default(), lockfile, maybe_config_file, maybe_import_map, maybe_inspector_server, root_cert_store: Some(root_cert_store.clone()), blob_store, broadcast_channel, shared_array_buffer_store, compiled_wasm_module_store, }))) } /// Prepares a set of module specifiers for loading in one shot. pub async fn prepare_module_graph( &self, specifiers: Vec, lib: TypeLib, root_permissions: Permissions, dynamic_permissions: Permissions, maybe_import_map: Option, ) -> Result<(), AnyError> { let handler = Arc::new(Mutex::new(FetchHandler::new( self, root_permissions, dynamic_permissions, )?)); let mut builder = GraphBuilder::new(handler, maybe_import_map, self.lockfile.clone()); for specifier in specifiers { builder.add(&specifier, false).await?; } builder.analyze_config_file(&self.maybe_config_file).await?; let mut graph = builder.get_graph(); let debug = self.flags.log_level == Some(log::Level::Debug); let maybe_config_file = self.maybe_config_file.clone(); let reload_exclusions = { let modules = self.modules.lock(); modules.keys().cloned().collect::>() }; let result_modules = if self.flags.no_check { let result_info = graph.transpile(TranspileOptions { debug, maybe_config_file, reload: self.flags.reload, reload_exclusions, })?; debug!("{}", result_info.stats); if let Some(ignored_options) = result_info.maybe_ignored_options { warn!("{}", ignored_options); } result_info.loadable_modules } else { let result_info = graph.check(CheckOptions { debug, emit: true, lib, maybe_config_file, reload: self.flags.reload, reload_exclusions, })?; debug!("{}", result_info.stats); if let Some(ignored_options) = result_info.maybe_ignored_options { eprintln!("{}", ignored_options); } if !result_info.diagnostics.is_empty() { return Err(anyhow!(result_info.diagnostics)); } result_info.loadable_modules }; let mut loadable_modules = self.modules.lock(); loadable_modules.extend(result_modules); if let Some(ref lockfile) = self.lockfile { let g = lockfile.lock(); g.write()?; } Ok(()) } /// This function is called when new module load is initialized by the JsRuntime. Its /// resposibility is to collect all dependencies and if it is required then also perform TS /// typecheck and traspilation. pub async fn prepare_module_load( &self, specifier: ModuleSpecifier, lib: TypeLib, root_permissions: Permissions, dynamic_permissions: Permissions, is_dynamic: bool, maybe_import_map: Option, ) -> Result<(), AnyError> { let specifier = specifier.clone(); let handler = Arc::new(Mutex::new(FetchHandler::new( self, root_permissions, dynamic_permissions, )?)); let mut builder = GraphBuilder::new(handler, maybe_import_map, self.lockfile.clone()); if self.flags.compat { builder.add(&compat::get_node_globals_url(), false).await?; } builder.add(&specifier, is_dynamic).await?; builder.analyze_config_file(&self.maybe_config_file).await?; let mut graph = builder.get_graph(); let debug = self.flags.log_level == Some(log::Level::Debug); let maybe_config_file = self.maybe_config_file.clone(); let reload_exclusions = { let modules = self.modules.lock(); modules.keys().cloned().collect::>() }; let result_modules = if self.flags.no_check { let result_info = graph.transpile(TranspileOptions { debug, maybe_config_file, reload: self.flags.reload, reload_exclusions, })?; debug!("{}", result_info.stats); if let Some(ignored_options) = result_info.maybe_ignored_options { warn!("{}", ignored_options); } result_info.loadable_modules } else { let result_info = graph.check(CheckOptions { debug, emit: true, lib, maybe_config_file, reload: self.flags.reload, reload_exclusions, })?; debug!("{}", result_info.stats); if let Some(ignored_options) = result_info.maybe_ignored_options { eprintln!("{}", ignored_options); } if !result_info.diagnostics.is_empty() { return Err(anyhow!(result_info.diagnostics)); } result_info.loadable_modules }; let mut loadable_modules = self.modules.lock(); loadable_modules.extend(result_modules); if let Some(ref lockfile) = self.lockfile { let g = lockfile.lock(); g.write()?; } Ok(()) } pub fn load( &self, specifier: ModuleSpecifier, maybe_referrer: Option, ) -> Result { let modules = self.modules.lock(); modules .get(&specifier) .map(|r| match r { Ok(module_source) => Ok(module_source.clone()), Err(err) => { // TODO(@kitsonk) this feels a bit hacky but it works, without // introducing another enum to have to try to deal with. if get_custom_error_class(err) == Some("NotFound") { let message = if let Some(referrer) = &maybe_referrer { format!("{}\n From: {}\n If the source module contains only types, use `import type` and `export type` to import it instead.", err, referrer) } else { format!("{}\n If the source module contains only types, use `import type` and `export type` to import it instead.", err) }; warn!("{}: {}", crate::colors::yellow("warning"), message); Ok(ModuleSource { code: "".to_string(), module_url_found: specifier.to_string(), module_url_specified: specifier.to_string(), }) } else { // anyhow errors don't support cloning, so we have to manage this // ourselves Err(anyhow!(err.to_string())) } }, }) .unwrap_or_else(|| { if let Some(referrer) = maybe_referrer { Err(anyhow!( "Module \"{}\" is missing from the graph.\n From: {}", specifier, referrer )) } else { Err(anyhow!( "Module \"{}\" is missing from the graph.", specifier )) } }) } // TODO(@kitsonk) this should be refactored to get it from the module graph fn get_emit(&self, url: &Url) -> Option<(Vec, Option>)> { match url.scheme() { // we should only be looking for emits for schemes that denote external // modules, which the disk_cache supports "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => { return None; } } let emit_path = self .dir .gen_cache .get_cache_filename_with_extension(url, "js")?; let emit_map_path = self .dir .gen_cache .get_cache_filename_with_extension(url, "js.map")?; if let Ok(code) = self.dir.gen_cache.get(&emit_path) { let maybe_map = if let Ok(map) = self.dir.gen_cache.get(&emit_map_path) { Some(map) } else { None }; Some((code, maybe_map)) } else { None } } } // TODO(@kitsonk) this is only temporary, but should be refactored to somewhere // else, like a refactored file_fetcher. impl SourceMapGetter for ProcState { fn get_source_map(&self, file_name: &str) -> Option> { if let Ok(specifier) = resolve_url(file_name) { if let Some((code, maybe_map)) = self.get_emit(&specifier) { let code = String::from_utf8(code).unwrap(); source_map_from_code(code).or(maybe_map) } else if let Ok(source) = self.load(specifier, None) { source_map_from_code(source.code) } else { None } } else { None } } fn get_source_line( &self, file_name: &str, line_number: usize, ) -> Option { if let Ok(specifier) = resolve_url(file_name) { self.file_fetcher.get_source(&specifier).map(|out| { // Do NOT use .lines(): it skips the terminating empty line. // (due to internally using .split_terminator() instead of .split()) let lines: Vec<&str> = out.source.split('\n').collect(); assert!(lines.len() > line_number); lines[line_number].to_string() }) } else { None } } } fn source_map_from_code(code: String) -> Option> { let lines: Vec<&str> = code.split('\n').collect(); if let Some(last_line) = lines.last() { if last_line .starts_with("//# sourceMappingURL=data:application/json;base64,") { let input = last_line.trim_start_matches( "//# sourceMappingURL=data:application/json;base64,", ); let decoded_map = base64::decode(input) .expect("Unable to decode source map from emitted file."); Some(decoded_map) } else { None } } else { None } }