denoland-deno/cli/cache/mod.rs

// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.

use crate::args::jsr_url;
use crate::args::CacheSetting;
use crate::errors::get_error_class_name;
use crate::file_fetcher::FetchNoFollowOptions;
use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
use crate::npm::CliNpmResolver;
use crate::resolver::CliNodeResolver;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::path::specifier_has_extension;
use crate::util::text_encoding::arc_str_to_bytes;
use crate::util::text_encoding::from_utf8_lossy_owned;

use deno_ast::MediaType;
use deno_core::futures;
use deno_core::futures::FutureExt;
use deno_core::ModuleSpecifier;
use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_runtime::deno_permissions::PermissionsContainer;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::SystemTime;

mod cache_db;
mod caches;
mod check;
mod code_cache;
mod common;
mod deno_dir;
mod disk_cache;
mod emit;
mod fast_check;
mod incremental;
mod module_info;
mod node;
mod parsed_source;

pub use cache_db::CacheDBHash;
pub use caches::Caches;
pub use check::TypeCheckCache;
pub use code_cache::CodeCache;
pub use common::FastInsecureHasher;
pub use deno_dir::dirs::home_dir;
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
pub use emit::EmitCache;
pub use fast_check::FastCheckCache;
pub use incremental::IncrementalCache;
pub use module_info::ModuleInfoCache;
pub use node::NodeAnalysisCache;
pub use parsed_source::EsmOrCjsChecker;
pub use parsed_source::LazyGraphSourceParser;
pub use parsed_source::ParsedSourceCache;

/// Permissions used to save a file in the disk caches.
pub const CACHE_PERM: u32 = 0o644;

#[derive(Debug, Clone)]
pub struct RealDenoCacheEnv;

impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
  fn read_file_bytes(&self, path: &Path) -> std::io::Result<Vec<u8>> {
    std::fs::read(path)
  }

  fn atomic_write_file(
    &self,
    path: &Path,
    bytes: &[u8],
  ) -> std::io::Result<()> {
    atomic_write_file_with_retries(path, bytes, CACHE_PERM)
  }

  fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
    crate::util::fs::canonicalize_path(path)
  }

  fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
    std::fs::create_dir_all(path)
  }

  fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
    match std::fs::metadata(path) {
      Ok(metadata) => Ok(Some(
        metadata.modified().unwrap_or_else(|_| SystemTime::now()),
      )),
      Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
      Err(err) => Err(err),
    }
  }

  fn is_file(&self, path: &Path) -> bool {
    path.is_file()
  }

  fn time_now(&self) -> SystemTime {
    SystemTime::now()
  }
}

#[derive(Debug, Clone)]
pub struct DenoCacheEnvFsAdapter<'a>(
  pub &'a dyn deno_runtime::deno_fs::FileSystem,
);

impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> {
  fn read_file_bytes(&self, path: &Path) -> std::io::Result<Vec<u8>> {
    self
      .0
      .read_file_sync(path, None)
      .map_err(|err| err.into_io_error())
  }

  fn atomic_write_file(
    &self,
    path: &Path,
    bytes: &[u8],
  ) -> std::io::Result<()> {
    atomic_write_file_with_retries_and_fs(
      &AtomicWriteFileFsAdapter {
        fs: self.0,
        write_mode: CACHE_PERM,
      },
      path,
      bytes,
    )
  }

  fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
    self.0.realpath_sync(path).map_err(|e| e.into_io_error())
  }

  fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
    self
      .0
      .mkdir_sync(path, true, None)
      .map_err(|e| e.into_io_error())
  }

  fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
    self
      .0
      .stat_sync(path)
      .map(|stat| {
        stat
          .mtime
          .map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts))
      })
      .map_err(|e| e.into_io_error())
  }

  fn is_file(&self, path: &Path) -> bool {
    self.0.is_file_sync(path)
  }

  fn time_now(&self) -> SystemTime {
    SystemTime::now()
  }
}

pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
pub type LocalLspHttpCache =
  deno_cache_dir::LocalLspHttpCache<RealDenoCacheEnv>;
pub use deno_cache_dir::HttpCache;

pub struct FetchCacherOptions {
  pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
  pub permissions: PermissionsContainer,
  /// If we're publishing for `deno publish`.
  pub is_deno_publish: bool,
  pub unstable_detect_cjs: bool,
}

/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
/// a concise interface to the DENO_DIR when building module graphs.
pub struct FetchCacher {
  pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
  esm_or_cjs_checker: Arc<EsmOrCjsChecker>,
  file_fetcher: Arc<FileFetcher>,
  global_http_cache: Arc<GlobalHttpCache>,
  node_resolver: Arc<CliNodeResolver>,
  npm_resolver: Arc<dyn CliNpmResolver>,
  module_info_cache: Arc<ModuleInfoCache>,
  permissions: PermissionsContainer,
  is_deno_publish: bool,
  unstable_detect_cjs: bool,
  cache_info_enabled: bool,
}

impl FetchCacher {
  pub fn new(
    esm_or_cjs_checker: Arc<EsmOrCjsChecker>,
    file_fetcher: Arc<FileFetcher>,
    global_http_cache: Arc<GlobalHttpCache>,
    node_resolver: Arc<CliNodeResolver>,
    npm_resolver: Arc<dyn CliNpmResolver>,
    module_info_cache: Arc<ModuleInfoCache>,
    options: FetchCacherOptions,
  ) -> Self {
    Self {
      file_fetcher,
      esm_or_cjs_checker,
      global_http_cache,
      node_resolver,
      npm_resolver,
      module_info_cache,
      file_header_overrides: options.file_header_overrides,
      permissions: options.permissions,
      is_deno_publish: options.is_deno_publish,
      unstable_detect_cjs: options.unstable_detect_cjs,
      cache_info_enabled: false,
    }
  }

  /// The cache information takes a bit of time to fetch and it's
  /// not always necessary. It should only be enabled for deno info.
  pub fn enable_loading_cache_info(&mut self) {
    self.cache_info_enabled = true;
  }

  /// Only use this for `deno info`.
  fn get_local_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> {
    // TODO(@kitsonk) fix when deno_graph does not query cache for synthetic
    // modules
    if specifier.scheme() == "flags" {
      None
    } else if specifier.scheme() == "file" {
      specifier.to_file_path().ok()
    } else {
      #[allow(deprecated)]
      self
        .global_http_cache
        .get_global_cache_filepath(specifier)
        .ok()
    }
  }
}

impl Loader for FetchCacher {
  fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option<CacheInfo> {
    if !self.cache_info_enabled {
      return None;
    }

    #[allow(deprecated)]
    let local = self.get_local_path(specifier)?;
    if local.is_file() {
      Some(CacheInfo { local: Some(local) })
    } else {
      None
    }
  }

  fn load(
    &self,
    specifier: &ModuleSpecifier,
    options: deno_graph::source::LoadOptions,
  ) -> LoadFuture {
    use deno_graph::source::CacheSetting as LoaderCacheSetting;

    if specifier.scheme() == "file" {
      if specifier.path().contains("/node_modules/") {
        // The specifier might be in a completely different symlinked tree than
        // what the node_modules url is in (ex. `/my-project-1/node_modules`
        // symlinked to `/my-project-2/node_modules`), so first we checked if the path
        // is in a node_modules dir to avoid needlessly canonicalizing, then now compare
        // against the canonicalized specifier.
        let specifier =
          crate::node::resolve_specifier_into_node_modules(specifier);
        if self.npm_resolver.in_npm_package(&specifier) {
          return Box::pin(futures::future::ready(Ok(Some(
            LoadResponse::External { specifier },
          ))));
        }
      }

      // make local CJS modules external to the graph
      if specifier_has_extension(specifier, "cjs") {
        return Box::pin(futures::future::ready(Ok(Some(
          LoadResponse::External {
            specifier: specifier.clone(),
          },
        ))));
      }

      if self.unstable_detect_cjs && specifier_has_extension(specifier, "js") {
        if let Ok(Some(pkg_json)) =
          self.node_resolver.get_closest_package_json(specifier)
        {
          if pkg_json.typ == "commonjs" {
            if let Ok(path) = specifier.to_file_path() {
              if let Ok(bytes) = std::fs::read(&path) {
                let text: Arc<str> = from_utf8_lossy_owned(bytes).into();
                let is_es_module = match self.esm_or_cjs_checker.is_esm(
                  specifier,
                  text.clone(),
                  MediaType::JavaScript,
                ) {
                  Ok(value) => value,
                  Err(err) => {
                    return Box::pin(futures::future::ready(Err(err.into())));
                  }
                };
                if !is_es_module {
                  self.node_resolver.mark_cjs_resolution(specifier.clone());
                  return Box::pin(futures::future::ready(Ok(Some(
                    LoadResponse::External {
                      specifier: specifier.clone(),
                    },
                  ))));
                } else {
                  return Box::pin(futures::future::ready(Ok(Some(
                    LoadResponse::Module {
                      specifier: specifier.clone(),
                      content: arc_str_to_bytes(text),
                      maybe_headers: None,
                    },
                  ))));
                }
              }
            }
          }
        }
      }
    }

    if self.is_deno_publish
      && matches!(specifier.scheme(), "http" | "https")
      && !specifier.as_str().starts_with(jsr_url().as_str())
    {
      // mark non-JSR remote modules as external so we don't need --allow-import
      // permissions as these will error out later when publishing
      return Box::pin(futures::future::ready(Ok(Some(
        LoadResponse::External {
          specifier: specifier.clone(),
        },
      ))));
    }

    let file_fetcher = self.file_fetcher.clone();
    let file_header_overrides = self.file_header_overrides.clone();
    let permissions = self.permissions.clone();
    let specifier = specifier.clone();
    let is_statically_analyzable = !options.was_dynamic_root;

    async move {
      let maybe_cache_setting = match options.cache_setting {
        LoaderCacheSetting::Use => None,
        LoaderCacheSetting::Reload => {
          if matches!(file_fetcher.cache_setting(), CacheSetting::Only) {
            return Err(deno_core::anyhow::anyhow!(
              "Could not resolve version constraint using only cached data. Try running again without --cached-only"
            ));
          }
          Some(CacheSetting::ReloadAll)
        }
        LoaderCacheSetting::Only => Some(CacheSetting::Only),
      };
      file_fetcher
        .fetch_no_follow_with_options(FetchNoFollowOptions {
          fetch_options: FetchOptions {
            specifier: &specifier,
            permissions: if is_statically_analyzable {
              FetchPermissionsOptionRef::StaticContainer(&permissions)
            } else {
              FetchPermissionsOptionRef::DynamicContainer(&permissions)
            },
            maybe_auth: None,
            maybe_accept: None,
            maybe_cache_setting: maybe_cache_setting.as_ref(),
          },
          maybe_checksum: options.maybe_checksum.as_ref(),
        })
        .await
        .map(|file_or_redirect| {
          match file_or_redirect {
            FileOrRedirect::File(file) => {
              let maybe_headers =
              match (file.maybe_headers, file_header_overrides.get(&specifier)) {
                (Some(headers), Some(overrides)) => {
                  Some(headers.into_iter().chain(overrides.clone()).collect())
                }
                (Some(headers), None) => Some(headers),
                (None, Some(overrides)) => Some(overrides.clone()),
                (None, None) => None,
              };
            Ok(Some(LoadResponse::Module {
              specifier: file.specifier,
              maybe_headers,
              content: file.source,
            }))
            },
            FileOrRedirect::Redirect(redirect_specifier) => {
              Ok(Some(LoadResponse::Redirect {
                specifier: redirect_specifier,
              }))
            },
          }
        })
        .unwrap_or_else(|err| {
          if let Some(io_err) = err.downcast_ref::<std::io::Error>() {
            if io_err.kind() == std::io::ErrorKind::NotFound {
              return Ok(None);
            } else {
              return Err(err);
            }
          }
          let error_class_name = get_error_class_name(&err);
          match error_class_name {
            "NotFound" => Ok(None),
            "NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None),
            _ => Err(err),
          }
        })
    }
    .boxed_local()
  }

  fn cache_module_info(
    &self,
    specifier: &ModuleSpecifier,
    media_type: MediaType,
    source: &Arc<[u8]>,
    module_info: &deno_graph::ModuleInfo,
  ) {
    log::debug!("Caching module info for {}", specifier);
    let source_hash = CacheDBHash::from_source(source);
    let result = self.module_info_cache.set_module_info(
      specifier,
      media_type,
      source_hash,
      module_info,
    );
    if let Err(err) = result {
      log::debug!(
        "Error saving module cache info for {}. {:#}",
        specifier,
        err
      );
    }
  }
}