foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-28 16:20:57 -05:00
Code Issues Packages 1 Wiki Activity

refactor: add EmitCache trait (#14925)

Browse source
This commit is contained in:
David Sherret 2022-06-20 17:59:52 -04:00 committed by GitHub
parent a7a64438e2
commit a7339f756c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 242 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
cli/cache.rs
View file

@ -8,7 +8,6 @@ use deno_core::error::AnyError;
use deno_core::futures::FutureExt;
use deno_core::serde::Deserialize;
use deno_core::serde::Serialize;
use deno_core::serde_json;
use deno_core::ModuleSpecifier;
use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
@ -40,22 +39,13 @@ pub trait Cacher {
) -> Option<String>;
/// Set a value in the cache.
fn set(
&mut self,
&self,
cache_type: CacheType,
specifier: &ModuleSpecifier,
value: String,
) -> Result<(), AnyError>;
}
/// Combines the cacher trait along with the deno_graph Loader trait to provide
/// a single interface to be able to load and cache modules when building a
/// graph.
pub trait CacherLoader: Cacher + Loader {
fn as_cacher(&self) -> &dyn Cacher;
fn as_mut_loader(&mut self) -> &mut dyn Loader;
fn as_mut_cacher(&mut self) -> &mut dyn Cacher;
}
/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
/// a concise interface to the DENO_DIR when building module graphs.
pub struct FetchCacher {
@ -81,30 +71,6 @@ impl FetchCacher {
root_permissions,
}
}
fn get_emit_metadata(
&self,
specifier: &ModuleSpecifier,
) -> Option<EmitMetadata> {
let filename = self
.disk_cache
.get_cache_filename_with_extension(specifier, "meta")?;
let bytes = self.disk_cache.get(&filename).ok()?;
serde_json::from_slice(&bytes).ok()
}
fn set_emit_metadata(
&self,
specifier: &ModuleSpecifier,
data: EmitMetadata,
) -> Result<(), AnyError> {
let filename = self
.disk_cache
.get_cache_filename_with_extension(specifier, "meta")
.unwrap();
let bytes = serde_json::to_vec(&data)?;
self.disk_cache.set(&filename, &bytes).map_err(|e| e.into())
}
}
impl Loader for FetchCacher {
@ -172,74 +138,3 @@ impl Loader for FetchCacher {
.boxed()
}
}
impl Cacher for FetchCacher {
fn get(
&self,
cache_type: CacheType,
specifier: &ModuleSpecifier,
) -> Option<String> {
let extension = match cache_type {
CacheType::Emit => "js",
CacheType::SourceMap => "js.map",
CacheType::TypeScriptBuildInfo => "buildinfo",
CacheType::Version => {
return self.get_emit_metadata(specifier).map(|d| d.version_hash)
}
};
let filename = self
.disk_cache
.get_cache_filename_with_extension(specifier, extension)?;
self
.disk_cache
.get(&filename)
.ok()
.and_then(|b| String::from_utf8(b).ok())
}
fn set(
&mut self,
cache_type: CacheType,
specifier: &ModuleSpecifier,
value: String,
) -> Result<(), AnyError> {
let extension = match cache_type {
CacheType::Emit => "js",
CacheType::SourceMap => "js.map",
CacheType::TypeScriptBuildInfo => "buildinfo",
CacheType::Version => {
let data = if let Some(mut data) = self.get_emit_metadata(specifier) {
data.version_hash = value;
data
} else {
EmitMetadata {
version_hash: value,
}
};
return self.set_emit_metadata(specifier, data);
}
};
let filename = self
.disk_cache
.get_cache_filename_with_extension(specifier, extension)
.unwrap();
self
.disk_cache
.set(&filename, value.as_bytes())
.map_err(|e| e.into())
}
}
impl CacherLoader for FetchCacher {
fn as_cacher(&self) -> &dyn Cacher {
self
}
fn as_mut_loader(&mut self) -> &mut dyn Loader {
self
}
fn as_mut_cacher(&mut self) -> &mut dyn Cacher {
self
}
}

79
cli/disk_cache.rs
View file

@ -1,7 +1,13 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::cache::CacheType;
use crate::cache::Cacher;
use crate::cache::EmitMetadata;
use crate::fs_util;
use crate::http_cache::url_to_filename;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::url::{Host, Url};
use std::ffi::OsStr;
use std::fs;
@ -145,6 +151,79 @@ impl DiskCache {
fs_util::atomic_write_file(&path, data, crate::http_cache::CACHE_PERM)
.map_err(|e| with_io_context(&e, format!("{:#?}", &path)))
}
fn get_emit_metadata(
&self,
specifier: &ModuleSpecifier,
) -> Option<EmitMetadata> {
let filename = self.get_cache_filename_with_extension(specifier, "meta")?;
let bytes = self.get(&filename).ok()?;
serde_json::from_slice(&bytes).ok()
}
fn set_emit_metadata(
&self,
specifier: &ModuleSpecifier,
data: EmitMetadata,
) -> Result<(), AnyError> {
let filename = self
.get_cache_filename_with_extension(specifier, "meta")
.unwrap();
let bytes = serde_json::to_vec(&data)?;
self.set(&filename, &bytes).map_err(|e| e.into())
}
}
// todo(13302): remove and replace with sqlite database
impl Cacher for DiskCache {
fn get(
&self,
cache_type: CacheType,
specifier: &ModuleSpecifier,
) -> Option<String> {
let extension = match cache_type {
CacheType::Emit => "js",
CacheType::SourceMap => "js.map",
CacheType::TypeScriptBuildInfo => "buildinfo",
CacheType::Version => {
return self.get_emit_metadata(specifier).map(|d| d.version_hash)
}
};
let filename =
self.get_cache_filename_with_extension(specifier, extension)?;
self
.get(&filename)
.ok()
.and_then(|b| String::from_utf8(b).ok())
}
fn set(
&self,
cache_type: CacheType,
specifier: &ModuleSpecifier,
value: String,
) -> Result<(), AnyError> {
let extension = match cache_type {
CacheType::Emit => "js",
CacheType::SourceMap => "js.map",
CacheType::TypeScriptBuildInfo => "buildinfo",
CacheType::Version => {
let data = if let Some(mut data) = self.get_emit_metadata(specifier) {
data.version_hash = value;
data
} else {
EmitMetadata {
version_hash: value,
}
};
return self.set_emit_metadata(specifier, data);
}
};
let filename = self
.get_cache_filename_with_extension(specifier, extension)
.unwrap();
self.set(&filename, value.as_bytes()).map_err(|e| e.into())
}
}
#[cfg(test)]

176
cli/emit.rs
View file

@ -43,6 +43,90 @@ use std::result;
use std::sync::Arc;
use std::time::Instant;
/// Emit cache for a single file.
#[derive(Debug, Clone, PartialEq)]
pub struct SpecifierEmitCacheData {
pub source_hash: String,
pub text: String,
pub map: Option<String>,
}
pub trait EmitCache {
/// Gets the emit data from the cache.
fn get_emit_data(
&self,
specifier: &ModuleSpecifier,
) -> Option<SpecifierEmitCacheData>;
/// Gets the stored hash of the source of the provider specifier
/// to tell if the emit is out of sync with the source.
/// TODO(13302): this is actually not reliable and should be removed
/// once switching to an sqlite db
fn get_source_hash(&self, specifier: &ModuleSpecifier) -> Option<String>;
/// Gets the emitted JavaScript of the TypeScript source.
/// TODO(13302): remove this once switching to an sqlite db
fn get_emit_text(&self, specifier: &ModuleSpecifier) -> Option<String>;
/// Sets the emit data in the cache.
fn set_emit_data(
&self,
specifier: ModuleSpecifier,
data: SpecifierEmitCacheData,
) -> Result<(), AnyError>;
/// Gets the .tsbuildinfo file from the cache.
fn get_tsbuildinfo(&self, specifier: &ModuleSpecifier) -> Option<String>;
/// Sets the .tsbuildinfo file in the cache.
fn set_tsbuildinfo(
&self,
specifier: ModuleSpecifier,
text: String,
) -> Result<(), AnyError>;
}
impl<T: Cacher> EmitCache for T {
fn get_emit_data(
&self,
specifier: &ModuleSpecifier,
) -> Option<SpecifierEmitCacheData> {
Some(SpecifierEmitCacheData {
source_hash: self.get_source_hash(specifier)?,
text: self.get_emit_text(specifier)?,
map: self.get(CacheType::SourceMap, specifier),
})
}
fn get_source_hash(&self, specifier: &ModuleSpecifier) -> Option<String> {
self.get(CacheType::Version, specifier)
}
fn get_emit_text(&self, specifier: &ModuleSpecifier) -> Option<String> {
self.get(CacheType::Emit, specifier)
}
fn set_emit_data(
&self,
specifier: ModuleSpecifier,
data: SpecifierEmitCacheData,
) -> Result<(), AnyError> {
self.set(CacheType::Version, &specifier, data.source_hash)?;
self.set(CacheType::Emit, &specifier, data.text)?;
if let Some(map) = data.map {
self.set(CacheType::SourceMap, &specifier, map)?;
}
Ok(())
}
fn get_tsbuildinfo(&self, specifier: &ModuleSpecifier) -> Option<String> {
self.get(CacheType::TypeScriptBuildInfo, specifier)
}
fn set_tsbuildinfo(
&self,
specifier: ModuleSpecifier,
text: String,
) -> Result<(), AnyError> {
self.set(CacheType::TypeScriptBuildInfo, &specifier, text)
}
}
/// Represents the "default" type library that should be used when type
/// checking the code in the module graph. Note that a user provided config
/// of `"lib"` would override this value.
@ -325,7 +409,7 @@ pub struct CheckEmitResult {
pub fn check_and_maybe_emit(
roots: &[(ModuleSpecifier, ModuleKind)],
graph_data: Arc<RwLock<GraphData>>,
cache: &mut dyn Cacher,
cache: &dyn EmitCache,
options: CheckOptions,
) -> Result<CheckEmitResult, AnyError> {
let check_js = options.ts_config.get_check_js();
@ -358,7 +442,7 @@ pub fn check_and_maybe_emit(
let maybe_tsbuildinfo = if options.reload {
None
} else {
cache.get(CacheType::TypeScriptBuildInfo, &roots[0].0)
cache.get_tsbuildinfo(&roots[0].0)
};
// to make tsc build info work, we need to consistently hash modules, so that
// tsc can better determine if an emit is still valid or not, so we provide
@ -400,9 +484,29 @@ pub fn check_and_maybe_emit(
// while we retrieve the build info for just the first module, it can be
// used for all the roots in the graph, so we will cache it for all roots
for (root, _) in roots {
cache.set(CacheType::TypeScriptBuildInfo, root, info.clone())?;
cache.set_tsbuildinfo(root.clone(), info.to_string())?;
}
}
struct SpecifierEmitData {
pub version_hash: String,
pub text: Option<String>,
pub map: Option<String>,
}
impl SpecifierEmitData {
fn into_cache_data(self) -> Option<SpecifierEmitCacheData> {
self.text.map(|text| SpecifierEmitCacheData {
source_hash: self.version_hash,
text,
map: self.map,
})
}
}
// combine the emitted files into groups based on their specifier and media type
let mut emit_data_items: HashMap<ModuleSpecifier, SpecifierEmitData> =
HashMap::with_capacity(response.emitted_files.len());
for emit in response.emitted_files.into_iter() {
if let Some(specifiers) = emit.maybe_specifiers {
assert!(specifiers.len() == 1);
@ -437,14 +541,21 @@ pub fn check_and_maybe_emit(
log::debug!("skipping emit for {}", specifier);
continue;
}
let mut emit_data_item = emit_data_items
.entry(specifier.clone())
.or_insert_with(|| SpecifierEmitData {
version_hash: get_version(source_bytes, &config_bytes),
text: None,
map: None,
});
match emit.media_type {
MediaType::JavaScript | MediaType::Mjs | MediaType::Cjs => {
let version = get_version(source_bytes, &config_bytes);
cache.set(CacheType::Version, &specifier, version)?;
cache.set(CacheType::Emit, &specifier, emit.data)?;
emit_data_item.text = Some(emit.data);
}
MediaType::SourceMap => {
cache.set(CacheType::SourceMap, &specifier, emit.data)?;
emit_data_item.map = Some(emit.data);
}
_ => unreachable!(
"unexpected media_type {} {}",
@ -453,6 +564,13 @@ pub fn check_and_maybe_emit(
}
}
}
// now insert these items into the cache
for (specifier, data) in emit_data_items.into_iter() {
if let Some(cache_data) = data.into_cache_data() {
cache.set_emit_data(specifier, cache_data)?;
}
}
}
Ok(CheckEmitResult {
@ -472,7 +590,7 @@ pub struct EmitOptions {
// `check_and_maybe_emit()`, but the AST isn't stored in that. Cleanup.
pub fn emit(
graph: &ModuleGraph,
cache: &mut dyn Cacher,
cache: &dyn EmitCache,
options: EmitOptions,
) -> Result<CheckEmitResult, AnyError> {
let start = Instant::now();
@ -493,15 +611,9 @@ pub fn emit(
module.maybe_source.as_ref().map(|s| s.as_bytes()).unwrap(),
&config_bytes,
);
let is_valid =
cache
.get(CacheType::Version, &module.specifier)
.map_or(false, |v| {
v == get_version(
module.maybe_source.as_ref().map(|s| s.as_bytes()).unwrap(),
&config_bytes,
)
});
let is_valid = cache
.get_source_hash(&module.specifier)
.map_or(false, |v| v == version);
if is_valid && !needs_reload {
continue;
}
@ -511,13 +623,14 @@ pub fn emit(
.map(|ps| ps.transpile(&emit_options))
.unwrap()?;
emit_count += 1;
cache.set(CacheType::Emit, &module.specifier, transpiled_source.text)?;
if let Some(map) = transpiled_source.source_map {
cache.set(CacheType::SourceMap, &module.specifier, map)?;
}
if !is_valid {
cache.set(CacheType::Version, &module.specifier, version)?;
}
cache.set_emit_data(
module.specifier.clone(),
SpecifierEmitCacheData {
source_hash: version,
text: transpiled_source.text,
map: transpiled_source.source_map,
},
)?;
}
let stats = Stats(vec![
@ -538,7 +651,7 @@ pub fn emit(
/// a valid emit in the cache.
fn valid_emit(
graph_data: &GraphData,
cache: &dyn Cacher,
cache: &dyn EmitCache,
ts_config: &TsConfig,
reload: bool,
reload_exclusions: &HashSet<ModuleSpecifier>,
@ -564,13 +677,20 @@ fn valid_emit(
continue;
}
}
_ => continue,
MediaType::Json
| MediaType::TsBuildInfo
| MediaType::SourceMap
| MediaType::Dts
| MediaType::Dmts
| MediaType::Dcts
| MediaType::Wasm
| MediaType::Unknown => continue,
}
if reload && !reload_exclusions.contains(specifier) {
return false;
}
if let Some(version) = cache.get(CacheType::Version, specifier) {
if version != get_version(code.as_bytes(), &config_bytes) {
if let Some(source_hash) = cache.get_source_hash(specifier) {
if source_hash != get_version(code.as_bytes(), &config_bytes) {
return false;
}
} else {

3
cli/lsp/cache.rs
View file

@ -1,6 +1,5 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::cache::CacherLoader;
use crate::cache::FetchCacher;
use crate::config_file::ConfigFile;
use crate::flags::Flags;
@ -87,7 +86,7 @@ impl CacheServer {
roots,
false,
maybe_imports.clone(),
cache.as_mut_loader(),
&mut cache,
maybe_resolver,
None,
None,

2
cli/main.rs
View file

@ -719,7 +719,7 @@ async fn create_graph_and_maybe_check(
let check_result = emit::check_and_maybe_emit(
&graph.roots,
Arc::new(RwLock::new(graph.as_ref().into())),
&mut cache,
&ps.dir.gen_cache,
emit::CheckOptions {
type_check_mode: ps.flags.type_check_mode.clone(),
debug,

46
cli/proc_state.rs
View file

@ -9,6 +9,7 @@ use crate::config_file::ConfigFile;
use crate::config_file::MaybeImportsResult;
use crate::deno_dir;
use crate::emit;
use crate::emit::EmitCache;
use crate::file_fetcher::get_root_cert_store;
use crate::file_fetcher::CacheSetting;
use crate::file_fetcher::FileFetcher;
@ -499,7 +500,7 @@ impl ProcState {
reload: self.flags.reload,
reload_exclusions,
};
let emit_result = emit::emit(&graph, &mut cache, options)?;
let emit_result = emit::emit(&graph, &self.dir.gen_cache, options)?;
log::debug!("{}", emit_result.stats);
} else {
let maybe_config_specifier = self
@ -519,7 +520,7 @@ impl ProcState {
let emit_result = emit::check_and_maybe_emit(
&roots,
self.graph_data.clone(),
&mut cache,
&self.dir.gen_cache,
options,
)?;
if !emit_result.diagnostics.is_empty() {
@ -633,16 +634,10 @@ impl ProcState {
| MediaType::Cts
| MediaType::Jsx
| MediaType::Tsx => {
let emit_path = self
.dir
.gen_cache
.get_cache_filename_with_extension(&found_url, "js")
.unwrap_or_else(|| {
unreachable!("Unable to get cache filename: {}", &found_url)
});
match self.dir.gen_cache.get(&emit_path) {
Ok(b) => String::from_utf8(b).unwrap(),
Err(_) => unreachable!("Unexpected missing emit: {}\n\nTry reloading with the --reload CLI flag or deleting your DENO_DIR.", found_url),
let cached_text = self.dir.gen_cache.get_emit_text(&found_url);
match cached_text {
Some(text) => text,
None => unreachable!("Unexpected missing emit: {}\n\nTry reloading with the --reload CLI flag or deleting your DENO_DIR.", found_url),
}
}
MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {
@ -665,28 +660,6 @@ impl ProcState {
)),
}
}
// TODO(@kitsonk) this should be refactored to get it from the module graph
fn get_emit(&self, url: &Url) -> Option<(Vec<u8>, Option<Vec<u8>>)> {
let emit_path = self
.dir
.gen_cache
.get_cache_filename_with_extension(url, "js")?;
let emit_map_path = self
.dir
.gen_cache
.get_cache_filename_with_extension(url, "js.map")?;
if let Ok(code) = self.dir.gen_cache.get(&emit_path) {
let maybe_map = if let Ok(map) = self.dir.gen_cache.get(&emit_map_path) {
Some(map)
} else {
None
};
Some((code, maybe_map))
} else {
None
}
}
}
// TODO(@kitsonk) this is only temporary, but should be refactored to somewhere
@ -700,8 +673,9 @@ impl SourceMapGetter for ProcState {
"wasm" | "file" | "http" | "https" | "data" | "blob" => (),
_ => return None,
}
if let Some((code, maybe_map)) = self.get_emit(&specifier) {
source_map_from_code(&code).or(maybe_map)
if let Some(cache_data) = self.dir.gen_cache.get_emit_data(&specifier) {
source_map_from_code(cache_data.text.as_bytes())
.or_else(|| cache_data.map.map(|t| t.into_bytes()))
} else if let Ok(source) = self.load(specifier, None, false) {
source_map_from_code(&source.code)
} else {

3
cli/tools/bench.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::cache;
use crate::cache::CacherLoader;
use crate::colors;
use crate::compat;
use crate::create_main_worker;
@ -632,7 +631,7 @@ pub async fn run_benchmarks_with_watch(
.collect(),
false,
maybe_imports,
cache.as_mut_loader(),
&mut cache,
maybe_resolver,
maybe_locker,
None,

3
cli/tools/test.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::cache;
use crate::cache::CacherLoader;
use crate::colors;
use crate::compat;
use crate::create_main_worker;
@ -1453,7 +1452,7 @@ pub async fn run_tests_with_watch(
.collect(),
false,
maybe_imports,
cache.as_mut_loader(),
&mut cache,
maybe_resolver,
maybe_locker,
None,