denoland-deno/cli/proc_state.rs

// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.

use crate::cache;
use crate::colors;
use crate::compat;
use crate::config_file::ConfigFile;
use crate::deno_dir;
use crate::emit;
use crate::errors::get_module_graph_error_class;
use crate::file_fetcher::CacheSetting;
use crate::file_fetcher::FileFetcher;
use crate::flags;
use crate::http_cache;
use crate::lockfile::as_maybe_locker;
use crate::lockfile::Lockfile;
use crate::resolver::ImportMapResolver;
use crate::source_maps::SourceMapGetter;
use crate::version;

use deno_core::error::anyhow;
use deno_core::error::custom_error;
use deno_core::error::get_custom_error_class;
use deno_core::error::AnyError;
use deno_core::error::Context;
use deno_core::parking_lot::Mutex;
use deno_core::resolve_url;
use deno_core::url::Url;
use deno_core::CompiledWasmModuleStore;
use deno_core::ModuleSource;
use deno_core::ModuleSpecifier;
use deno_core::SharedArrayBufferStore;
use deno_runtime::deno_broadcast_channel::InMemoryBroadcastChannel;
use deno_runtime::deno_web::BlobStore;
use deno_runtime::inspector_server::InspectorServer;
use deno_runtime::permissions::Permissions;
use deno_tls::rustls::RootCertStore;
use deno_tls::rustls_native_certs::load_native_certs;
use deno_tls::webpki_roots::TLS_SERVER_ROOTS;
use import_map::ImportMap;
use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::fs::File;
use std::io::BufReader;
use std::ops::Deref;
use std::sync::Arc;

/// This structure represents state of single "deno" program.
///
/// It is shared by all created workers (thus V8 isolates).
#[derive(Clone)]
pub struct ProcState(Arc<Inner>);

pub struct Inner {
  /// Flags parsed from `argv` contents.
  pub flags: flags::Flags,
  pub dir: deno_dir::DenoDir,
  pub coverage_dir: Option<String>,
  pub file_fetcher: FileFetcher,
  modules: Arc<Mutex<HashMap<ModuleSpecifier, Result<ModuleSource, AnyError>>>>,
  pub lockfile: Option<Arc<Mutex<Lockfile>>>,
  pub maybe_config_file: Option<ConfigFile>,
  pub maybe_import_map: Option<ImportMap>,
  pub maybe_inspector_server: Option<Arc<InspectorServer>>,
  // deno_graph detects all sorts of issues at build time (prepare_module_load)
  // but if they are errors at that stage, the don't cause the correct behaviors
  // so we cache the error and then surface it when appropriate (e.g. load)
  pub(crate) maybe_graph_error:
    Arc<Mutex<Option<deno_graph::ModuleGraphError>>>,
  // because the graph detects resolution issues early, but is build and dropped
  // during the `prepare_module_load` method, we need to extract out the module
  // resolution map so that those errors can be surfaced at the appropriate time
  resolution_map:
    Arc<Mutex<HashMap<ModuleSpecifier, HashMap<String, deno_graph::Resolved>>>>,
  // in some cases we want to provide the span where the resolution error
  // occurred but need to surface it on load, but on load we don't know who the
  // referrer and span was, so we need to cache those
  resolved_map: Arc<Mutex<HashMap<ModuleSpecifier, deno_graph::Span>>>,
  pub root_cert_store: Option<RootCertStore>,
  pub blob_store: BlobStore,
  pub broadcast_channel: InMemoryBroadcastChannel,
  pub shared_array_buffer_store: SharedArrayBufferStore,
  pub compiled_wasm_module_store: CompiledWasmModuleStore,
}

impl Deref for ProcState {
  type Target = Arc<Inner>;
  fn deref(&self) -> &Self::Target {
    &self.0
  }
}

impl ProcState {
  pub async fn build(flags: flags::Flags) -> Result<Self, AnyError> {
    let maybe_custom_root = flags
      .cache_path
      .clone()
      .or_else(|| env::var("DENO_DIR").map(String::into).ok());
    let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
    let deps_cache_location = dir.root.join("deps");
    let http_cache = http_cache::HttpCache::new(&deps_cache_location);

    let mut root_cert_store = RootCertStore::empty();
    let ca_stores: Vec<String> = flags
      .ca_stores
      .clone()
      .or_else(|| {
        let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?;
        Some(
          env_ca_store
            .split(',')
            .map(|s| s.trim().to_string())
            .filter(|s| !s.is_empty())
            .collect(),
        )
      })
      .unwrap_or_else(|| vec!["mozilla".to_string()]);

    for store in ca_stores.iter() {
      match store.as_str() {
        "mozilla" => {
          root_cert_store.add_server_trust_anchors(&TLS_SERVER_ROOTS);
        }
        "system" => {
          let roots = load_native_certs()
            .expect("could not load platform certs")
            .roots;
          root_cert_store.roots.extend(roots);
        }
        _ => {
          return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store));
        }
      }
    }

    let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok());
    if let Some(ca_file) = ca_file {
      let certfile = File::open(&ca_file)?;
      let mut reader = BufReader::new(certfile);

      // This function does not return specific errors, if it fails give a generic message.
      if let Err(_err) = root_cert_store.add_pem_file(&mut reader) {
        return Err(anyhow!("Unable to add pem file to certificate store"));
      }
    }

    if let Some(insecure_allowlist) =
      flags.unsafely_ignore_certificate_errors.as_ref()
    {
      let domains = if insecure_allowlist.is_empty() {
        "for all hostnames".to_string()
      } else {
        format!("for: {}", insecure_allowlist.join(", "))
      };
      let msg =
        format!("DANGER: TLS certificate validation is disabled {}", domains);
      eprintln!("{}", colors::yellow(msg));
    }

    let cache_usage = if flags.cached_only {
      CacheSetting::Only
    } else if !flags.cache_blocklist.is_empty() {
      CacheSetting::ReloadSome(flags.cache_blocklist.clone())
    } else if flags.reload {
      CacheSetting::ReloadAll
    } else {
      CacheSetting::Use
    };

    let blob_store = BlobStore::default();
    let broadcast_channel = InMemoryBroadcastChannel::default();
    let shared_array_buffer_store = SharedArrayBufferStore::default();
    let compiled_wasm_module_store = CompiledWasmModuleStore::default();

    let file_fetcher = FileFetcher::new(
      http_cache,
      cache_usage,
      !flags.no_remote,
      Some(root_cert_store.clone()),
      blob_store.clone(),
      flags.unsafely_ignore_certificate_errors.clone(),
    )?;

    let lockfile = if let Some(filename) = &flags.lock {
      let lockfile = Lockfile::new(filename.clone(), flags.lock_write)?;
      Some(Arc::new(Mutex::new(lockfile)))
    } else {
      None
    };

    let maybe_config_file =
      if let Some(config_path) = flags.config_path.as_ref() {
        Some(ConfigFile::read(config_path)?)
      } else {
        None
      };

    let mut maybe_import_map: Option<ImportMap> =
      match flags.import_map_path.as_ref() {
        None => None,
        Some(import_map_url) => {
          let import_map_specifier =
            deno_core::resolve_url_or_path(import_map_url).context(format!(
              "Bad URL (\"{}\") for import map.",
              import_map_url
            ))?;
          let file = file_fetcher
            .fetch(&import_map_specifier, &mut Permissions::allow_all())
            .await
            .context(format!(
              "Unable to load '{}' import map",
              import_map_specifier
            ))?;
          let import_map =
            ImportMap::from_json(import_map_specifier.as_str(), &file.source)?;
          Some(import_map)
        }
      };

    if flags.compat {
      let mut import_map = match maybe_import_map {
        Some(import_map) => import_map,
        None => {
          // INFO: we're creating an empty import map, with its specifier pointing
          // to `CWD/node_import_map.json` to make sure the map still works as expected.
          let import_map_specifier =
            std::env::current_dir()?.join("node_import_map.json");
          ImportMap::from_json(import_map_specifier.to_str().unwrap(), "{}")
            .unwrap()
        }
      };
      let node_builtins = compat::get_mapped_node_builtins();
      let diagnostics = import_map.update_imports(node_builtins)?;

      if !diagnostics.is_empty() {
        log::info!("Some Node built-ins were not added to the import map:");
        for diagnostic in diagnostics {
          log::info!("  - {}", diagnostic);
        }
        log::info!("If you want to use Node built-ins provided by Deno remove listed specifiers from \"imports\" mapping in the import map file.");
      }

      maybe_import_map = Some(import_map);
    }

    let maybe_inspect_host = flags.inspect.or(flags.inspect_brk);
    let maybe_inspector_server = maybe_inspect_host.map(|host| {
      Arc::new(InspectorServer::new(host, version::get_user_agent()))
    });

    let coverage_dir = flags
      .coverage_dir
      .clone()
      .or_else(|| env::var("DENO_UNSTABLE_COVERAGE_DIR").ok());

    Ok(ProcState(Arc::new(Inner {
      dir,
      coverage_dir,
      flags,
      file_fetcher,
      modules: Default::default(),
      lockfile,
      maybe_config_file,
      maybe_import_map,
      maybe_inspector_server,
      maybe_graph_error: Default::default(),
      resolution_map: Default::default(),
      resolved_map: Default::default(),
      root_cert_store: Some(root_cert_store.clone()),
      blob_store,
      broadcast_channel,
      shared_array_buffer_store,
      compiled_wasm_module_store,
    })))
  }

  /// Return any imports that should be brought into the scope of the module
  /// graph.
  fn get_maybe_imports(&self) -> Option<Vec<(ModuleSpecifier, Vec<String>)>> {
    let mut imports = Vec::new();
    if let Some(config_file) = &self.maybe_config_file {
      if let Some(config_imports) = config_file.to_maybe_imports() {
        imports.extend(config_imports);
      }
    }
    if self.flags.compat {
      imports.extend(compat::get_node_imports());
    }
    if imports.is_empty() {
      None
    } else {
      Some(imports)
    }
  }

  /// This method is called when a module requested by the `JsRuntime` is not
  /// available, or in other sub-commands that need to "load" a module graph.
  /// The method will collect all the dependencies of the provided specifier,
  /// optionally checks their integrity, optionally type checks them, and
  /// ensures that any modules that needs to be transpiled is transpiled.
  ///
  /// It then populates the `loadable_modules` with what can be loaded into v8.
  pub(crate) async fn prepare_module_load(
    &self,
    roots: Vec<ModuleSpecifier>,
    is_dynamic: bool,
    lib: emit::TypeLib,
    root_permissions: Permissions,
    dynamic_permissions: Permissions,
  ) -> Result<(), AnyError> {
    let mut cache = cache::FetchCacher::new(
      self.dir.gen_cache.clone(),
      self.file_fetcher.clone(),
      root_permissions.clone(),
      dynamic_permissions.clone(),
    );
    let maybe_locker = as_maybe_locker(self.lockfile.clone());
    let maybe_imports = self.get_maybe_imports();
    let maybe_resolver =
      self.maybe_import_map.as_ref().map(ImportMapResolver::new);
    let graph = deno_graph::create_graph(
      roots,
      is_dynamic,
      maybe_imports,
      &mut cache,
      maybe_resolver.as_ref().map(|im| im.as_resolver()),
      maybe_locker,
      None,
    )
    .await;
    // If there was a locker, validate the integrity of all the modules in the
    // locker.
    emit::lock(&graph);

    // Determine any modules that have already been emitted this session and
    // should be skipped.
    let reload_exclusions: HashSet<ModuleSpecifier> = {
      let modules = self.modules.lock();
      modules.keys().cloned().collect()
    };

    let config_type = if self.flags.no_check {
      emit::ConfigType::Emit
    } else {
      emit::ConfigType::Check {
        tsc_emit: true,
        lib,
      }
    };

    let (ts_config, maybe_ignored_options) =
      emit::get_ts_config(config_type, self.maybe_config_file.as_ref(), None)?;
    let graph = Arc::new(graph);

    // we will store this in proc state later, as if we were to return it from
    // prepare_load, some dynamic errors would not be catchable
    let maybe_graph_error = graph.valid().err();

    if emit::valid_emit(
      graph.as_ref(),
      &cache,
      &ts_config,
      self.flags.reload,
      &reload_exclusions,
    ) {
      if let Some(root) = graph.roots.get(0) {
        log::debug!("specifier \"{}\" and dependencies have valid emit, skipping checking and emitting", root);
      } else {
        log::debug!("rootless graph, skipping checking and emitting");
      }
    } else {
      if let Some(ignored_options) = maybe_ignored_options {
        log::warn!("{}", ignored_options);
      }
      let emit_result = if self.flags.no_check {
        let options = emit::EmitOptions {
          ts_config,
          reload_exclusions,
          reload: self.flags.reload,
        };
        emit::emit(graph.as_ref(), &mut cache, options)?
      } else {
        // here, we are type checking, so we want to error here if any of the
        // type only dependencies are missing or we have other errors with them
        // where as if we are not type checking, we shouldn't care about these
        // errors, and they don't get returned in `graph.valid()` above.
        graph.valid_types_only()?;

        let maybe_config_specifier = self
          .maybe_config_file
          .as_ref()
          .map(|cf| ModuleSpecifier::from_file_path(&cf.path).unwrap());
        let options = emit::CheckOptions {
          debug: self.flags.log_level == Some(log::Level::Debug),
          emit_with_diagnostics: true,
          maybe_config_specifier,
          ts_config,
        };
        for root in &graph.roots {
          let root_str = root.to_string();
          // `$deno$` specifiers are internal specifiers, printing out that
          // they are being checked is confusing to a user, since they don't
          // actually exist, so we will simply indicate that a generated module
          // is being checked instead of the cryptic internal module
          if !root_str.contains("$deno$") {
            log::info!("{} {}", colors::green("Check"), root);
          } else {
            log::info!("{} a generated module", colors::green("Check"))
          }
        }
        emit::check_and_maybe_emit(graph.clone(), &mut cache, options)?
      };
      log::debug!("{}", emit_result.stats);
      // if the graph is not valid then the diagnostics returned are bogus and
      // should just be ignored so that module loading can proceed to allow the
      // "real" error to be surfaced
      if !emit_result.diagnostics.is_empty() && maybe_graph_error.is_none() {
        return Err(anyhow!(emit_result.diagnostics));
      }
    }

    // we iterate over the graph, looking for any modules that were emitted, or
    // should be loaded as their un-emitted source and add them to the in memory
    // cache of modules for loading by deno_core.
    {
      let mut modules = self.modules.lock();
      modules.extend(emit::to_module_sources(graph.as_ref(), &cache));
    }

    // since we can't store the graph in proc state, because proc state needs to
    // be thread safe because of the need to provide source map resolution and
    // the graph needs to not be thread safe (due to wasmbind_gen constraints),
    // we have no choice but to extract out other meta data from the graph to
    // provide the correct loading behaviors for CLI
    {
      let mut resolution_map = self.resolution_map.lock();
      resolution_map.extend(graph.resolution_map());
    }
    {
      let mut self_maybe_graph_error = self.maybe_graph_error.lock();
      *self_maybe_graph_error = maybe_graph_error;
    }

    // any updates to the lockfile should be updated now
    if let Some(ref lockfile) = self.lockfile {
      let g = lockfile.lock();
      g.write()?;
    }

    Ok(())
  }

  pub(crate) fn resolve(
    &self,
    specifier: &str,
    referrer: &str,
  ) -> Result<ModuleSpecifier, AnyError> {
    let resolution_map = self.resolution_map.lock();
    if let Some((_, Some(map))) = deno_core::resolve_url_or_path(referrer)
      .ok()
      .map(|s| (s.clone(), resolution_map.get(&s)))
    {
      if let Some(resolved) = map.get(specifier) {
        match resolved {
          Some(Ok((specifier, span))) => {
            let mut resolved_map = self.resolved_map.lock();
            resolved_map.insert(specifier.clone(), span.clone());
            return Ok(specifier.clone());
          }
          Some(Err(err)) => {
            return Err(custom_error(
              "TypeError",
              format!("{}\n", err.to_string_with_span()),
            ))
          }
          _ => (),
        }
      }
    }
    // FIXME(bartlomieju): hacky way to provide compatibility with repl
    let referrer = if referrer.is_empty() && self.flags.repl {
      deno_core::DUMMY_SPECIFIER
    } else {
      referrer
    };
    if let Some(import_map) = &self.maybe_import_map {
      import_map
        .resolve(specifier, referrer)
        .map_err(|err| err.into())
    } else {
      deno_core::resolve_import(specifier, referrer).map_err(|err| err.into())
    }
  }

  pub fn load(
    &self,
    specifier: ModuleSpecifier,
    maybe_referrer: Option<ModuleSpecifier>,
    is_dynamic: bool,
  ) -> Result<ModuleSource, AnyError> {
    log::debug!(
      "specifier: {} maybe_referrer: {} is_dynamic: {}",
      specifier,
      maybe_referrer
        .as_ref()
        .map(|s| s.to_string())
        .unwrap_or_else(|| "<none>".to_string()),
      is_dynamic
    );
    let modules = self.modules.lock();
    modules
      .get(&specifier)
      .map(|r| match r {
        Ok(module_source) => Ok(module_source.clone()),
        Err(err) => {
          // this is the "pending" error we will return
          let err = if let Some(error_class) = get_custom_error_class(err) {
            if error_class == "NotFound" && maybe_referrer.is_some() && !is_dynamic {
              let resolved_map = self.resolved_map.lock();
              // in situations where we were to try to load a module that wasn't
              // emitted and we can't run the original source code (it isn't)
              // JavaScript, we will load a blank module instead.  This is
              // usually caused by people exporting type only exports and not
              // type checking.
              if let Some(span) = resolved_map.get(&specifier) {
                log::warn!("{}: Cannot load module \"{}\".\n    at {}\n  If the source module contains only types, use `import type` and `export type` to import it instead.", colors::yellow("warning"), specifier, span);
                return Ok(ModuleSource {
                  code: "".to_string(),
                  module_url_found: specifier.to_string(),
                  module_url_specified: specifier.to_string(),
                });
              }
            }
            custom_error(error_class, err.to_string())
          } else {
            anyhow!(err.to_string())
          };
          // if there is a pending graph error though we haven't returned, we
          // will return that one
          let mut maybe_graph_error = self.maybe_graph_error.lock();
          if let Some(graph_error) = maybe_graph_error.take() {
            log::debug!("returning cached graph error");
            let resolved_map = self.resolved_map.lock();
            if let Some(span) = resolved_map.get(&specifier) {
              if !span.specifier.as_str().contains("$deno") {
                return Err(custom_error(get_module_graph_error_class(&graph_error), format!("{}\n    at {}", graph_error, span)));
              }
            }
            Err(graph_error.into())
          } else {
            Err(err)
          }
        }
      })
      .unwrap_or_else(|| {
        if maybe_referrer.is_some() && !is_dynamic {
          let resolved_map = self.resolved_map.lock();
          if let Some(span) = resolved_map.get(&specifier) {
            return Err(custom_error("NotFound", format!("Cannot load module \"{}\".\n    at {}", specifier, span)));
          }
        }
        Err(custom_error("NotFound", format!("Cannot load module \"{}\".", specifier)))
      })
  }

  // TODO(@kitsonk) this should be refactored to get it from the module graph
  fn get_emit(&self, url: &Url) -> Option<(Vec<u8>, Option<Vec<u8>>)> {
    match url.scheme() {
      // we should only be looking for emits for schemes that denote external
      // modules, which the disk_cache supports
      "wasm" | "file" | "http" | "https" | "data" | "blob" => (),
      _ => {
        return None;
      }
    }
    let emit_path = self
      .dir
      .gen_cache
      .get_cache_filename_with_extension(url, "js")?;
    let emit_map_path = self
      .dir
      .gen_cache
      .get_cache_filename_with_extension(url, "js.map")?;
    if let Ok(code) = self.dir.gen_cache.get(&emit_path) {
      let maybe_map = if let Ok(map) = self.dir.gen_cache.get(&emit_map_path) {
        Some(map)
      } else {
        None
      };
      Some((code, maybe_map))
    } else {
      None
    }
  }
}

// TODO(@kitsonk) this is only temporary, but should be refactored to somewhere
// else, like a refactored file_fetcher.
impl SourceMapGetter for ProcState {
  fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
    if let Ok(specifier) = resolve_url(file_name) {
      if let Some((code, maybe_map)) = self.get_emit(&specifier) {
        let code = String::from_utf8(code).unwrap();
        source_map_from_code(code).or(maybe_map)
      } else if let Ok(source) = self.load(specifier, None, false) {
        source_map_from_code(source.code)
      } else {
        None
      }
    } else {
      None
    }
  }

  fn get_source_line(
    &self,
    file_name: &str,
    line_number: usize,
  ) -> Option<String> {
    if let Ok(specifier) = resolve_url(file_name) {
      self.file_fetcher.get_source(&specifier).map(|out| {
        // Do NOT use .lines(): it skips the terminating empty line.
        // (due to internally using .split_terminator() instead of .split())
        let lines: Vec<&str> = out.source.split('\n').collect();
        assert!(lines.len() > line_number);
        lines[line_number].to_string()
      })
    } else {
      None
    }
  }
}

fn source_map_from_code(code: String) -> Option<Vec<u8>> {
  let lines: Vec<&str> = code.split('\n').collect();
  if let Some(last_line) = lines.last() {
    if last_line
      .starts_with("//# sourceMappingURL=data:application/json;base64,")
    {
      let input = last_line.trim_start_matches(
        "//# sourceMappingURL=data:application/json;base64,",
      );
      let decoded_map = base64::decode(input)
        .expect("Unable to decode source map from emitted file.");
      Some(decoded_map)
    } else {
      None
    }
  } else {
    None
  }
}