2023-01-02 16:00:42 -05:00
|
|
|
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
|
2021-10-10 17:26:22 -04:00
|
|
|
|
2021-12-22 08:25:06 -05:00
|
|
|
use crate::errors::get_error_class_name;
|
2021-10-10 17:26:22 -04:00
|
|
|
use crate::file_fetcher::FileFetcher;
|
|
|
|
|
2023-02-22 17:21:05 -05:00
|
|
|
use deno_core::futures;
|
2021-10-10 17:26:22 -04:00
|
|
|
use deno_core::futures::FutureExt;
|
|
|
|
use deno_core::ModuleSpecifier;
|
|
|
|
use deno_graph::source::CacheInfo;
|
|
|
|
use deno_graph::source::LoadFuture;
|
|
|
|
use deno_graph::source::LoadResponse;
|
|
|
|
use deno_graph::source::Loader;
|
2023-01-07 11:25:34 -05:00
|
|
|
use deno_runtime::permissions::PermissionsContainer;
|
2023-03-22 10:15:53 -04:00
|
|
|
use std::collections::HashMap;
|
2023-08-01 20:49:09 -04:00
|
|
|
use std::path::PathBuf;
|
2021-10-10 17:26:22 -04:00
|
|
|
use std::sync::Arc;
|
|
|
|
|
feat(core): initialize SQLite off-main-thread (#18401)
This gets SQLite off the flamegraph and reduces initialization time by
somewhere between 0.2ms and 0.5ms. In addition, I took the opportunity
to move all the cache management code to a single place and reduce
duplication. While the PR has a net gain of lines, much of that is just
being a bit more deliberate with how we're recovering from errors.
The existing caches had various policies for dealing with cache
corruption, so I've unified them and tried to isolate the decisions we
make for recovery in a single place (see `open_connection` in
`CacheDB`). The policy I chose was:
1. Retry twice to open on-disk caches
2. If that fails, try to delete the file and recreate it on-disk
3. If we fail to delete the file or re-create a new cache, use a
fallback strategy that can be chosen per-cache: InMemory (temporary
cache for the process run), BlackHole (ignore writes, return empty
reads), or Error (fail on every operation).
The caches all use the same general code now, and share the cache
failure recovery policy.
In addition, it cleans up a TODO in the `NodeAnalysisCache`.
2023-03-27 18:01:52 -04:00
|
|
|
mod cache_db;
|
|
|
|
mod caches;
|
2022-07-12 18:58:39 -04:00
|
|
|
mod check;
|
|
|
|
mod common;
|
2022-11-25 19:04:30 -05:00
|
|
|
mod deno_dir;
|
2022-07-12 18:58:39 -04:00
|
|
|
mod disk_cache;
|
|
|
|
mod emit;
|
2022-11-28 17:28:54 -05:00
|
|
|
mod http_cache;
|
2022-07-12 18:58:39 -04:00
|
|
|
mod incremental;
|
2022-10-01 06:15:56 -04:00
|
|
|
mod node;
|
2022-08-22 12:14:59 -04:00
|
|
|
mod parsed_source;
|
2022-07-12 18:58:39 -04:00
|
|
|
|
feat(core): initialize SQLite off-main-thread (#18401)
This gets SQLite off the flamegraph and reduces initialization time by
somewhere between 0.2ms and 0.5ms. In addition, I took the opportunity
to move all the cache management code to a single place and reduce
duplication. While the PR has a net gain of lines, much of that is just
being a bit more deliberate with how we're recovering from errors.
The existing caches had various policies for dealing with cache
corruption, so I've unified them and tried to isolate the decisions we
make for recovery in a single place (see `open_connection` in
`CacheDB`). The policy I chose was:
1. Retry twice to open on-disk caches
2. If that fails, try to delete the file and recreate it on-disk
3. If we fail to delete the file or re-create a new cache, use a
fallback strategy that can be chosen per-cache: InMemory (temporary
cache for the process run), BlackHole (ignore writes, return empty
reads), or Error (fail on every operation).
The caches all use the same general code now, and share the cache
failure recovery policy.
In addition, it cleans up a TODO in the `NodeAnalysisCache`.
2023-03-27 18:01:52 -04:00
|
|
|
pub use caches::Caches;
|
2022-07-12 18:58:39 -04:00
|
|
|
pub use check::TypeCheckCache;
|
2022-07-19 11:58:18 -04:00
|
|
|
pub use common::FastInsecureHasher;
|
2022-11-25 19:04:30 -05:00
|
|
|
pub use deno_dir::DenoDir;
|
2023-05-25 14:27:45 -04:00
|
|
|
pub use deno_dir::DenoDirProvider;
|
2022-07-12 18:58:39 -04:00
|
|
|
pub use disk_cache::DiskCache;
|
|
|
|
pub use emit::EmitCache;
|
2022-11-28 17:28:54 -05:00
|
|
|
pub use http_cache::CachedUrlMetadata;
|
2023-08-01 20:49:09 -04:00
|
|
|
pub use http_cache::GlobalHttpCache;
|
2022-11-28 17:28:54 -05:00
|
|
|
pub use http_cache::HttpCache;
|
2023-08-01 20:49:09 -04:00
|
|
|
pub use http_cache::LocalHttpCache;
|
2023-08-02 16:57:25 -04:00
|
|
|
pub use http_cache::LocalLspHttpCache;
|
2022-07-12 18:58:39 -04:00
|
|
|
pub use incremental::IncrementalCache;
|
2022-10-01 06:15:56 -04:00
|
|
|
pub use node::NodeAnalysisCache;
|
2022-08-22 12:14:59 -04:00
|
|
|
pub use parsed_source::ParsedSourceCache;
|
2022-07-12 18:58:39 -04:00
|
|
|
|
2022-11-28 17:28:54 -05:00
|
|
|
/// Permissions used to save a file in the disk caches.
|
|
|
|
pub const CACHE_PERM: u32 = 0o644;
|
|
|
|
|
2021-10-10 17:26:22 -04:00
|
|
|
/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
|
|
|
|
/// a concise interface to the DENO_DIR when building module graphs.
|
2022-03-23 09:54:22 -04:00
|
|
|
pub struct FetchCacher {
|
2022-07-19 11:58:18 -04:00
|
|
|
emit_cache: EmitCache,
|
2021-10-10 17:26:22 -04:00
|
|
|
file_fetcher: Arc<FileFetcher>,
|
2023-03-22 10:15:53 -04:00
|
|
|
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
|
2023-08-01 20:49:09 -04:00
|
|
|
global_http_cache: Arc<GlobalHttpCache>,
|
2023-04-26 16:23:28 -04:00
|
|
|
permissions: PermissionsContainer,
|
2023-02-08 19:45:04 -05:00
|
|
|
cache_info_enabled: bool,
|
2023-02-22 17:21:05 -05:00
|
|
|
maybe_local_node_modules_url: Option<ModuleSpecifier>,
|
2021-10-10 17:26:22 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
impl FetchCacher {
|
|
|
|
pub fn new(
|
2022-07-19 11:58:18 -04:00
|
|
|
emit_cache: EmitCache,
|
2023-02-03 14:15:16 -05:00
|
|
|
file_fetcher: Arc<FileFetcher>,
|
2023-03-22 10:15:53 -04:00
|
|
|
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
|
2023-08-01 20:49:09 -04:00
|
|
|
global_http_cache: Arc<GlobalHttpCache>,
|
2023-04-26 16:23:28 -04:00
|
|
|
permissions: PermissionsContainer,
|
2023-02-22 17:21:05 -05:00
|
|
|
maybe_local_node_modules_url: Option<ModuleSpecifier>,
|
2021-10-10 17:26:22 -04:00
|
|
|
) -> Self {
|
|
|
|
Self {
|
2022-07-19 11:58:18 -04:00
|
|
|
emit_cache,
|
2021-10-10 17:26:22 -04:00
|
|
|
file_fetcher,
|
2023-03-22 10:15:53 -04:00
|
|
|
file_header_overrides,
|
2023-08-01 20:49:09 -04:00
|
|
|
global_http_cache,
|
2023-04-26 16:23:28 -04:00
|
|
|
permissions,
|
2023-02-08 19:45:04 -05:00
|
|
|
cache_info_enabled: false,
|
2023-02-22 17:21:05 -05:00
|
|
|
maybe_local_node_modules_url,
|
2021-10-10 17:26:22 -04:00
|
|
|
}
|
|
|
|
}
|
2023-02-08 19:45:04 -05:00
|
|
|
|
|
|
|
/// The cache information takes a bit of time to fetch and it's
|
|
|
|
/// not always necessary. It should only be enabled for deno info.
|
|
|
|
pub fn enable_loading_cache_info(&mut self) {
|
|
|
|
self.cache_info_enabled = true;
|
|
|
|
}
|
2023-08-01 20:49:09 -04:00
|
|
|
|
|
|
|
// DEPRECATED: Where the file is stored and how it's stored should be an implementation
|
|
|
|
// detail of the cache.
|
|
|
|
//
|
|
|
|
// todo(dsheret): remove once implementing
|
|
|
|
// * https://github.com/denoland/deno/issues/17707
|
|
|
|
// * https://github.com/denoland/deno/issues/17703
|
|
|
|
#[deprecated(
|
|
|
|
note = "There should not be a way to do this because the file may not be cached at a local path in the future."
|
|
|
|
)]
|
|
|
|
fn get_local_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> {
|
|
|
|
// TODO(@kitsonk) fix when deno_graph does not query cache for synthetic
|
|
|
|
// modules
|
|
|
|
if specifier.scheme() == "flags" {
|
|
|
|
None
|
|
|
|
} else if specifier.scheme() == "file" {
|
|
|
|
specifier.to_file_path().ok()
|
|
|
|
} else {
|
|
|
|
#[allow(deprecated)]
|
|
|
|
self
|
|
|
|
.global_http_cache
|
|
|
|
.get_global_cache_filepath(specifier)
|
|
|
|
.ok()
|
|
|
|
}
|
|
|
|
}
|
2021-10-10 17:26:22 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Loader for FetchCacher {
|
|
|
|
fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option<CacheInfo> {
|
2023-02-08 19:45:04 -05:00
|
|
|
if !self.cache_info_enabled {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2023-07-08 16:06:45 -04:00
|
|
|
#[allow(deprecated)]
|
2023-08-01 20:49:09 -04:00
|
|
|
let local = self.get_local_path(specifier)?;
|
2021-10-10 17:26:22 -04:00
|
|
|
if local.is_file() {
|
|
|
|
let emit = self
|
2022-07-19 11:58:18 -04:00
|
|
|
.emit_cache
|
|
|
|
.get_emit_filepath(specifier)
|
2021-10-10 17:26:22 -04:00
|
|
|
.filter(|p| p.is_file());
|
|
|
|
Some(CacheInfo {
|
|
|
|
local: Some(local),
|
|
|
|
emit,
|
2022-07-19 11:58:18 -04:00
|
|
|
map: None,
|
2021-10-10 17:26:22 -04:00
|
|
|
})
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn load(
|
|
|
|
&mut self,
|
|
|
|
specifier: &ModuleSpecifier,
|
2023-04-26 16:23:28 -04:00
|
|
|
_is_dynamic: bool,
|
2021-10-10 17:26:22 -04:00
|
|
|
) -> LoadFuture {
|
2023-02-22 17:21:05 -05:00
|
|
|
if let Some(node_modules_url) = self.maybe_local_node_modules_url.as_ref() {
|
2023-02-23 10:58:10 -05:00
|
|
|
// The specifier might be in a completely different symlinked tree than
|
|
|
|
// what the resolved node_modules_url is in (ex. `/my-project-1/node_modules`
|
|
|
|
// symlinked to `/my-project-2/node_modules`), so first check if the path
|
|
|
|
// is in a node_modules dir to avoid needlessly canonicalizing, then compare
|
|
|
|
// against the canonicalized specifier.
|
|
|
|
if specifier.path().contains("/node_modules/") {
|
2023-02-24 19:35:43 -05:00
|
|
|
let specifier =
|
|
|
|
crate::node::resolve_specifier_into_node_modules(specifier);
|
2023-02-23 10:58:10 -05:00
|
|
|
if specifier.as_str().starts_with(node_modules_url.as_str()) {
|
|
|
|
return Box::pin(futures::future::ready(Ok(Some(
|
|
|
|
LoadResponse::External { specifier },
|
|
|
|
))));
|
|
|
|
}
|
2023-02-22 17:21:05 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-04-26 16:23:28 -04:00
|
|
|
let permissions = self.permissions.clone();
|
2021-10-10 17:26:22 -04:00
|
|
|
let file_fetcher = self.file_fetcher.clone();
|
2023-03-22 10:15:53 -04:00
|
|
|
let file_header_overrides = self.file_header_overrides.clone();
|
2023-02-22 14:15:25 -05:00
|
|
|
let specifier = specifier.clone();
|
2021-10-10 17:26:22 -04:00
|
|
|
|
|
|
|
async move {
|
2022-01-13 11:58:00 -05:00
|
|
|
file_fetcher
|
2023-01-07 11:25:34 -05:00
|
|
|
.fetch(&specifier, permissions)
|
2021-10-10 17:26:22 -04:00
|
|
|
.await
|
2023-03-15 17:46:36 -04:00
|
|
|
.map(|file| {
|
2023-03-22 10:15:53 -04:00
|
|
|
let maybe_headers =
|
|
|
|
match (file.maybe_headers, file_header_overrides.get(&specifier)) {
|
|
|
|
(Some(headers), Some(overrides)) => {
|
|
|
|
Some(headers.into_iter().chain(overrides.clone()).collect())
|
|
|
|
}
|
|
|
|
(Some(headers), None) => Some(headers),
|
|
|
|
(None, Some(overrides)) => Some(overrides.clone()),
|
|
|
|
(None, None) => None,
|
|
|
|
};
|
2023-03-15 17:46:36 -04:00
|
|
|
Ok(Some(LoadResponse::Module {
|
|
|
|
specifier: file.specifier,
|
2023-03-22 10:15:53 -04:00
|
|
|
maybe_headers,
|
2023-03-15 17:46:36 -04:00
|
|
|
content: file.source,
|
|
|
|
}))
|
|
|
|
})
|
|
|
|
.unwrap_or_else(|err| {
|
|
|
|
if let Some(err) = err.downcast_ref::<std::io::Error>() {
|
|
|
|
if err.kind() == std::io::ErrorKind::NotFound {
|
2021-12-22 08:25:06 -05:00
|
|
|
return Ok(None);
|
2021-10-10 17:26:22 -04:00
|
|
|
}
|
2023-03-15 17:46:36 -04:00
|
|
|
} else if get_error_class_name(&err) == "NotFound" {
|
|
|
|
return Ok(None);
|
|
|
|
}
|
|
|
|
Err(err)
|
|
|
|
})
|
2021-10-10 17:26:22 -04:00
|
|
|
}
|
|
|
|
.boxed()
|
|
|
|
}
|
|
|
|
}
|