From 852823fa505d75d61e70e1330bbf366aa248e650 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 19 Feb 2020 08:17:13 -0500 Subject: [PATCH] refactor: rewrite HTTP cache for file fetcher (#4030) --- cli/deno_dir.rs | 4 - cli/file_fetcher.rs | 618 ++++++------------------- cli/global_state.rs | 5 +- cli/http_cache.rs | 195 ++++++++ cli/http_util.rs | 176 +++---- cli/lib.rs | 3 +- cli/tests/022_info_flag_script.out | 2 +- cli/tests/049_info_flag_script_jsx.out | 2 +- cli/tests/cafile_info.ts.out | 2 +- cli/tests/integration_tests.rs | 21 +- 10 files changed, 428 insertions(+), 600 deletions(-) create mode 100644 cli/http_cache.rs diff --git a/cli/deno_dir.rs b/cli/deno_dir.rs index cac9c37393..44f97abbd2 100644 --- a/cli/deno_dir.rs +++ b/cli/deno_dir.rs @@ -10,8 +10,6 @@ use std::path::PathBuf; pub struct DenoDir { // Example: /Users/rld/.deno/ pub root: PathBuf, - /// Used by SourceFileFetcher to cache remote modules. - pub deps_cache: DiskCache, /// Used by TsCompiler to cache compiler output. pub gen_cache: DiskCache, } @@ -29,12 +27,10 @@ impl DenoDir { .unwrap_or(fallback); let root: PathBuf = custom_root.unwrap_or(default); - let deps_path = root.join("deps"); let gen_path = root.join("gen"); let deno_dir = Self { root, - deps_cache: DiskCache::new(&deps_path), gen_cache: DiskCache::new(&gen_path), }; diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 41f31fb493..de3a76eec3 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -3,11 +3,10 @@ use crate::colors; use crate::deno_error::DenoError; use crate::deno_error::ErrorKind; use crate::deno_error::GetErrorKind; -use crate::disk_cache::DiskCache; +use crate::http_cache::HttpCache; use crate::http_util; use crate::http_util::create_http_client; use crate::http_util::FetchOnceResult; -use crate::http_util::ResultPayload; use crate::msg; use deno_core::ErrBox; use deno_core::ModuleSpecifier; @@ -15,11 +14,11 @@ use futures::future::Either; use futures::future::FutureExt; use regex::Regex; use reqwest; -use serde_json; use std; use std::collections::HashMap; use std::fs; use std::future::Future; +use std::io::Read; use std::path::Path; use std::path::PathBuf; use std::pin::Pin; @@ -30,28 +29,6 @@ use std::sync::Mutex; use url; use url::Url; -pub fn source_header_cache_failed_error( - module_name: &str, - reason: &str, -) -> ErrBox { - DenoError::new( - ErrorKind::Other, - format!( - "Source code header cache failed for '{}': {}", - module_name, reason - ), - ) - .into() -} - -pub fn source_cache_failed_error(module_name: &str, reason: &str) -> ErrBox { - DenoError::new( - ErrorKind::Other, - format!("Source code cache failed for '{}': {}", module_name, reason), - ) - .into() -} - /// Structure representing local or remote file. /// /// In case of remote file `url` might be different than originally requested URL, if so @@ -90,22 +67,21 @@ impl SourceFileCache { const SUPPORTED_URL_SCHEMES: [&str; 3] = ["http", "https", "file"]; -/// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them -/// in single directory that can be controlled with `$DENO_DIR` env variable. #[derive(Clone)] pub struct SourceFileFetcher { - deps_cache: DiskCache, source_file_cache: SourceFileCache, cache_blacklist: Vec, use_disk_cache: bool, no_remote: bool, cached_only: bool, http_client: reqwest::Client, + // This field is public only to expose it's location + pub http_cache: HttpCache, } impl SourceFileFetcher { pub fn new( - deps_cache: DiskCache, + http_cache: HttpCache, use_disk_cache: bool, cache_blacklist: Vec, no_remote: bool, @@ -113,7 +89,7 @@ impl SourceFileFetcher { ca_file: Option, ) -> Result { let file_fetcher = Self { - deps_cache, + http_cache, source_file_cache: SourceFileCache::default(), cache_blacklist, use_disk_cache, @@ -333,54 +309,45 @@ impl SourceFileFetcher { &self, module_url: &Url, ) -> Result, ErrBox> { - let source_code_headers = self.get_source_code_headers(&module_url); - // If source code headers says that it would redirect elsewhere, - // (meaning that the source file might not exist; only .headers.json is present) - // Abort reading attempts to the cached source file and and follow the redirect. - if let Some(redirect_to) = source_code_headers.redirect_to { - // E.g. - // module_name https://import-meta.now.sh/redirect.js - // filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/redirect.js - // redirect_to https://import-meta.now.sh/sub/final1.js - // real_filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/sub/final1.js - // real_module_name = https://import-meta.now.sh/sub/final1.js - let redirect_url = Url::parse(&redirect_to).expect("Should be valid URL"); - - // Recurse. - // TODO(bartlomieju): I'm pretty sure we should call `fetch_remote_source_async` here. - // Should we expect that all redirects are cached? - return self.fetch_cached_remote_source(&redirect_url); - } - - // No redirect needed or end of redirects. - // We can try read the file - let filepath = self - .deps_cache - .location - .join(self.deps_cache.get_cache_filename(&module_url)); - let source_code = match fs::read(filepath.clone()) { + let result = self.http_cache.get(&module_url); + let result = match result { Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - return Ok(None); - } else { - return Err(e.into()); + if let Some(e) = e.downcast_ref::() { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(None); + } } + return Err(e); } Ok(c) => c, }; - let media_type = - map_content_type(&filepath, source_code_headers.mime_type.as_deref()); + + let (mut source_file, headers) = result; + if let Some(redirect_to) = headers.get("location") { + let redirect_url = Url::parse(redirect_to).expect("Should be valid URL"); + return self.fetch_cached_remote_source(&redirect_url); + } + + let mut source_code = Vec::new(); + source_file.read_to_end(&mut source_code)?; + + let cache_filename = self.http_cache.get_cache_filename(module_url); + let fake_filepath = PathBuf::from(module_url.path()); + let media_type = map_content_type( + &fake_filepath, + headers.get("content-type").map(|e| e.as_str()), + ); let types_url = match media_type { msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url( &module_url, &source_code, - source_code_headers.x_typescript_types.as_deref(), + headers.get("x-typescript-types").map(|e| e.as_str()), ), _ => None, }; Ok(Some(SourceFile { url: module_url.clone(), - filename: filepath, + filename: cache_filename, media_type, source_code, types_url, @@ -440,8 +407,10 @@ impl SourceFileFetcher { ); let dir = self.clone(); let module_url = module_url.clone(); - let headers = self.get_source_code_headers(&module_url); - let module_etag = headers.etag; + let module_etag = match self.http_cache.get(&module_url) { + Ok((_, headers)) => headers.get("etag").map(String::from), + Err(_) => None, + }; let http_client = self.http_client.clone(); // Single pass fetch, either yields code or yields redirect. let f = async move { @@ -453,20 +422,9 @@ impl SourceFileFetcher { Ok(source_file) } - FetchOnceResult::Redirect(new_module_url) => { + FetchOnceResult::Redirect(new_module_url, headers) => { // If redirects, update module_name and filename for next looped call. - if let Err(e) = dir.save_source_code_headers( - &module_url, - None, - Some(new_module_url.to_string()), - None, - None, - ) { - return Err(source_header_cache_failed_error( - module_url.as_str(), - &e.to_string(), - )); - } + dir.http_cache.set(&module_url, headers, &[])?; // Recurse dir @@ -478,51 +436,30 @@ impl SourceFileFetcher { ) .await } - FetchOnceResult::Code(ResultPayload { - body: source, - content_type: maybe_content_type, - etag, - x_typescript_types, - }) => { + FetchOnceResult::Code(source, headers) => { // We land on the code. - if let Err(e) = dir.save_source_code_headers( - &module_url, - maybe_content_type.clone(), - None, - etag, - x_typescript_types.clone(), - ) { - return Err(source_header_cache_failed_error( - module_url.as_str(), - &e.to_string(), - )); - } + dir.http_cache.set(&module_url, headers.clone(), &source)?; - if let Err(e) = dir.save_source_code(&module_url, &source) { - return Err(source_cache_failed_error( - module_url.as_str(), - &e.to_string(), - )); - } - - let filepath = dir - .deps_cache - .location - .join(dir.deps_cache.get_cache_filename(&module_url)); - - let media_type = - map_content_type(&filepath, maybe_content_type.as_deref()); + let cache_filepath = dir.http_cache.get_cache_filename(&module_url); + // Used to sniff out content type from file extension - probably to be removed + let fake_filepath = PathBuf::from(module_url.path()); + let media_type = map_content_type( + &fake_filepath, + headers.get("content-type").map(String::as_str), + ); let types_url = match media_type { - msg::MediaType::JavaScript | msg::MediaType::JSX => { - get_types_url(&module_url, &source, x_typescript_types.as_deref()) - } + msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url( + &module_url, + &source, + headers.get("x-typescript-types").map(String::as_str), + ), _ => None, }; let source_file = SourceFile { url: module_url.clone(), - filename: filepath, + filename: cache_filepath, media_type, source_code: source, types_url, @@ -535,75 +472,6 @@ impl SourceFileFetcher { f.boxed() } - - /// Get header metadata associated with a remote file. - /// - /// NOTE: chances are that the source file was downloaded due to redirects. - /// In this case, the headers file provides info about where we should go and get - /// the file that redirect eventually points to. - fn get_source_code_headers(&self, url: &Url) -> SourceCodeHeaders { - let cache_key = self - .deps_cache - .get_cache_filename_with_extension(url, "headers.json"); - - if let Ok(bytes) = self.deps_cache.get(&cache_key) { - if let Ok(json_string) = std::str::from_utf8(&bytes) { - return SourceCodeHeaders::from_json_string(json_string.to_string()); - } - } - - SourceCodeHeaders::default() - } - - /// Save contents of downloaded remote file in on-disk cache for subsequent access. - fn save_source_code(&self, url: &Url, source: &[u8]) -> std::io::Result<()> { - let cache_key = self.deps_cache.get_cache_filename(url); - - // May not exist. DON'T unwrap. - let _ = self.deps_cache.remove(&cache_key); - - self.deps_cache.set(&cache_key, source) - } - - /// Save headers related to source file to {filename}.headers.json file, - /// only when there is actually something necessary to save. - /// - /// For example, if the extension ".js" already mean JS file and we have - /// content type of "text/javascript", then we would not save the mime type. - /// - /// If nothing needs to be saved, the headers file is not created. - fn save_source_code_headers( - &self, - url: &Url, - mime_type: Option, - redirect_to: Option, - etag: Option, - x_typescript_types: Option, - ) -> std::io::Result<()> { - let cache_key = self - .deps_cache - .get_cache_filename_with_extension(url, "headers.json"); - - // Remove possibly existing stale .headers.json file. - // May not exist. DON'T unwrap. - let _ = self.deps_cache.remove(&cache_key); - - let headers = SourceCodeHeaders { - mime_type, - redirect_to, - etag, - x_typescript_types, - }; - - let cache_filename = self.deps_cache.get_cache_filename(url); - if let Ok(maybe_json_string) = headers.to_json_string(&cache_filename) { - if let Some(json_string) = maybe_json_string { - return self.deps_cache.set(&cache_key, json_string.as_bytes()); - } - } - - Ok(()) - } } fn map_file_extension(path: &Path) -> msg::MediaType { @@ -755,92 +623,14 @@ pub struct SourceCodeHeaders { pub x_typescript_types: Option, } -static MIME_TYPE: &str = "mime_type"; -static REDIRECT_TO: &str = "redirect_to"; -static ETAG: &str = "etag"; -static X_TYPESCRIPT_TYPES: &str = "x_typescript_types"; - -impl SourceCodeHeaders { - pub fn from_json_string(headers_string: String) -> Self { - // TODO: use serde for deserialization - let maybe_headers_json: serde_json::Result = - serde_json::from_str(&headers_string); - - if let Ok(headers_json) = maybe_headers_json { - let mime_type = headers_json[MIME_TYPE].as_str().map(String::from); - let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from); - let etag = headers_json[ETAG].as_str().map(String::from); - let x_typescript_types = - headers_json[X_TYPESCRIPT_TYPES].as_str().map(String::from); - - return SourceCodeHeaders { - mime_type, - redirect_to, - etag, - x_typescript_types, - }; - } - - SourceCodeHeaders::default() - } - - // TODO: remove this nonsense `cache_filename` param, this should be - // done when instantiating SourceCodeHeaders - pub fn to_json_string( - &self, - cache_filename: &Path, - ) -> Result, serde_json::Error> { - // TODO(kevinkassimo): consider introduce serde::Deserialize to make things simpler. - // This is super ugly at this moment... - // Had trouble to make serde_derive work: I'm unable to build proc-macro2. - let mut value_map = serde_json::map::Map::new(); - - if let Some(mime_type) = &self.mime_type { - let resolved_mime_type = - map_content_type(Path::new(""), Some(mime_type.clone().as_str())); - - // TODO: fix this - let ext_based_mime_type = map_file_extension(cache_filename); - - // Add mime to headers only when content type is different from extension. - if ext_based_mime_type == msg::MediaType::Unknown - || resolved_mime_type != ext_based_mime_type - { - value_map.insert(MIME_TYPE.to_string(), json!(mime_type)); - } - } - - if let Some(redirect_to) = &self.redirect_to { - value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to)); - } - - if let Some(etag) = &self.etag { - value_map.insert(ETAG.to_string(), json!(etag)); - } - - if let Some(x_typescript_types) = &self.x_typescript_types { - value_map - .insert(X_TYPESCRIPT_TYPES.to_string(), json!(x_typescript_types)); - } - - if value_map.is_empty() { - return Ok(None); - } - - serde_json::to_string(&value_map) - .and_then(|serialized| Ok(Some(serialized))) - } -} - #[cfg(test)] mod tests { use super::*; - use crate::fs as deno_fs; use tempfile::TempDir; fn setup_file_fetcher(dir_path: &Path) -> SourceFileFetcher { SourceFileFetcher::new( - DiskCache::new(&dir_path.to_path_buf().join("deps")), + HttpCache::new(&dir_path.to_path_buf().join("deps")).unwrap(), true, vec![], false, @@ -925,49 +715,6 @@ mod tests { assert_eq!(check_cache_blacklist(&u, &args), true); } - #[test] - fn test_source_code_headers_get_and_save() { - let (_temp_dir, fetcher) = test_setup(); - let url = Url::parse("http://example.com/f.js").unwrap(); - let headers_filepath = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&url, "headers.json"), - ); - - if let Some(ref parent) = headers_filepath.parent() { - fs::create_dir_all(parent).unwrap(); - }; - - let _ = deno_fs::write_file( - headers_filepath.as_path(), - "{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}", - 0o666, - ); - let headers = fetcher.get_source_code_headers(&url); - - assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript"); - assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js"); - assert_eq!(headers.etag, None); - assert_eq!(headers.x_typescript_types, None); - - let _ = fetcher.save_source_code_headers( - &url, - Some("text/typescript".to_owned()), - Some("http://deno.land/a.js".to_owned()), - Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()), - Some("./a.d.ts".to_owned()), - ); - let headers2 = fetcher.get_source_code_headers(&url); - assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript"); - assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js"); - assert_eq!( - headers2.etag.unwrap(), - "W/\"04572f4749af993f4961a7e5daa1e4d5\"" - ); - assert_eq!(headers2.x_typescript_types.unwrap(), "./a.d.ts") - } - #[test] fn test_fetch_local_file_no_panic() { let (_temp_dir, fetcher) = test_setup(); @@ -992,14 +739,12 @@ mod tests { Url::parse("http://localhost:4545/cli/tests/subdir/mod2.ts").unwrap(); let module_url_1 = module_url.clone(); let module_url_2 = module_url.clone(); - let headers_file_name = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&module_url, "headers.json"), - ); + let headers_file_name = fetcher + .http_cache + .get_cache_filename(&module_url) + .with_extension("headers.json"); let headers_file_name_1 = headers_file_name.clone(); let headers_file_name_2 = headers_file_name.clone(); - let headers_file_name_3 = headers_file_name; let result = fetcher .get_source_file_async(&module_url, true, false, false) @@ -1011,13 +756,12 @@ mod tests { &b"export { printHello } from \"./print_hello.ts\";\n"[..] ); assert_eq!(&(r.media_type), &msg::MediaType::TypeScript); - // Should not create .headers.json file due to matching ext - assert!(fs::read_to_string(&headers_file_name_1).is_err()); + assert!(fs::read_to_string(&headers_file_name_1).is_ok()); - // Modify .headers.json, write using fs write and read using save_source_code_headers + // Modify .headers.json, write using fs write let _ = fs::write( &headers_file_name_1, - "{ \"mime_type\": \"text/javascript\" }", + "{ \"content-type\": \"text/javascript\" }", ); let result2 = fetcher_1 .get_source_file_async(&module_url, true, false, false) @@ -1031,21 +775,14 @@ mod tests { // If get_source_file_async does not call remote, this should be JavaScript // as we modified before! (we do not overwrite .headers.json due to no http fetch) assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript); - assert_eq!( - fetcher_2 - .get_source_code_headers(&module_url_1) - .mime_type - .unwrap(), - "text/javascript" - ); + let (_, headers) = fetcher_2.http_cache.get(&module_url_1).unwrap(); + + assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); // Modify .headers.json again, but the other way around - let _ = fetcher_2.save_source_code_headers( - &module_url_1, - Some("application/json".to_owned()), - None, - None, - None, + let _ = fs::write( + &headers_file_name_1, + "{ \"content-type\": \"application/json\" }", ); let result3 = fetcher_2 .get_source_file_async(&module_url_1, true, false, false) @@ -1073,9 +810,8 @@ mod tests { let r4 = result4.unwrap(); let expected4 = &b"export { printHello } from \"./print_hello.ts\";\n"[..]; assert_eq!(r4.source_code, expected4); - // Now the old .headers.json file should have gone! Resolved back to TypeScript + // Resolved back to TypeScript assert_eq!(&(r4.media_type), &msg::MediaType::TypeScript); - assert!(fs::read_to_string(&headers_file_name_3).is_err()); drop(http_server_guard); } @@ -1084,17 +820,14 @@ mod tests { async fn test_get_source_code_2() { let http_server_guard = crate::test_util::http_server(); let (temp_dir, fetcher) = test_setup(); - let fetcher_1 = fetcher.clone(); let module_url = Url::parse("http://localhost:4545/cli/tests/subdir/mismatch_ext.ts") .unwrap(); let module_url_1 = module_url.clone(); - let module_url_2 = module_url.clone(); - let headers_file_name = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&module_url, "headers.json"), - ); + let headers_file_name = fetcher + .http_cache + .get_cache_filename(&module_url) + .with_extension("headers.json"); let result = fetcher .get_source_file_async(&module_url, true, false, false) @@ -1103,23 +836,14 @@ mod tests { let r = result.unwrap(); let expected = b"export const loaded = true;\n"; assert_eq!(r.source_code, expected); - // Mismatch ext with content type, create .headers.json assert_eq!(&(r.media_type), &msg::MediaType::JavaScript); - assert_eq!( - fetcher - .get_source_code_headers(&module_url) - .mime_type - .unwrap(), - "text/javascript" - ); + let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); + assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); // Modify .headers.json - let _ = fetcher.save_source_code_headers( - &module_url, - Some("text/typescript".to_owned()), - None, - None, - None, + let _ = fs::write( + &headers_file_name, + "{ \"content-type\": \"text/typescript\" }", ); let result2 = fetcher .get_source_file_async(&module_url, true, false, false) @@ -1132,9 +856,12 @@ mod tests { // as we modified before! (we do not overwrite .headers.json due to no http // fetch) assert_eq!(&(r2.media_type), &msg::MediaType::TypeScript); - assert!(fs::read_to_string(&headers_file_name).is_err()); + assert_eq!( + fs::read_to_string(&headers_file_name).unwrap(), + "{ \"content-type\": \"text/typescript\" }", + ); - // let's create fresh instance of DenoDir (simulating another freshh Deno + // let's create fresh instance of DenoDir (simulating another fresh Deno // process) and don't use cache let fetcher = setup_file_fetcher(temp_dir.path()); let result3 = fetcher @@ -1147,13 +874,8 @@ mod tests { // Now the old .headers.json file should be overwritten back to JavaScript! // (due to http fetch) assert_eq!(&(r3.media_type), &msg::MediaType::JavaScript); - assert_eq!( - fetcher_1 - .get_source_code_headers(&module_url_2) - .mime_type - .unwrap(), - "text/javascript" - ); + let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); + assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); drop(http_server_guard); } @@ -1166,11 +888,10 @@ mod tests { "http://localhost:4545/cli/tests/subdir/mismatch_ext.ts", ) .unwrap(); - let headers_file_name = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(specifier.as_url(), "headers.json"), - ); + let headers_file_name = fetcher + .http_cache + .get_cache_filename(specifier.as_url()) + .with_extension("headers.json"); // first download let r = fetcher.fetch_source_file_async(&specifier, None).await; @@ -1210,20 +931,16 @@ mod tests { "http://localhost:4546/cli/tests/subdir/redirects/redirect1.js", ) .unwrap(); - let redirect_source_filepath = fetcher - .deps_cache - .location - .join("http/localhost_PORT4546/cli/tests/subdir/redirects/redirect1.js"); + let redirect_source_filepath = + fetcher.http_cache.get_cache_filename(&redirect_module_url); let redirect_source_filename = redirect_source_filepath.to_str().unwrap().to_string(); let target_module_url = Url::parse( "http://localhost:4545/cli/tests/subdir/redirects/redirect1.js", ) .unwrap(); - let redirect_target_filepath = fetcher - .deps_cache - .location - .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js"); + let redirect_target_filepath = + fetcher.http_cache.get_cache_filename(&target_module_url); let redirect_target_filename = redirect_target_filepath.to_str().unwrap().to_string(); @@ -1233,13 +950,11 @@ mod tests { .await; assert!(result.is_ok()); let mod_meta = result.unwrap(); - // File that requires redirection is not downloaded. - assert!(fs::read_to_string(&redirect_source_filename).is_err()); - // ... but its .headers.json is created. - let redirect_source_headers = - fetcher.get_source_code_headers(&redirect_module_url); + // File that requires redirection should be empty file. + assert_eq!(fs::read_to_string(&redirect_source_filename).unwrap(), ""); + let (_, headers) = fetcher.http_cache.get(&redirect_module_url).unwrap(); assert_eq!( - redirect_source_headers.redirect_to.unwrap(), + headers.get("location").unwrap(), "http://localhost:4545/cli/tests/subdir/redirects/redirect1.js" ); // The target of redirection is downloaded instead. @@ -1247,10 +962,8 @@ mod tests { fs::read_to_string(&redirect_target_filename).unwrap(), "export const redirect = 1;\n" ); - let redirect_target_headers = - fetcher.get_source_code_headers(&target_module_url); - assert!(redirect_target_headers.redirect_to.is_none()); - + let (_, headers) = fetcher.http_cache.get(&target_module_url).unwrap(); + assert!(headers.get("location").is_none()); // Examine the meta result. assert_eq!(mod_meta.url, target_module_url); @@ -1265,28 +978,20 @@ mod tests { "http://localhost:4548/cli/tests/subdir/redirects/redirect1.js", ) .unwrap(); - let double_redirect_path = fetcher - .deps_cache - .location - .join("http/localhost_PORT4548/cli/tests/subdir/redirects/redirect1.js"); + let double_redirect_path = + fetcher.http_cache.get_cache_filename(&double_redirect_url); let redirect_url = Url::parse( "http://localhost:4546/cli/tests/subdir/redirects/redirect1.js", ) .unwrap(); - let redirect_path = fetcher - .deps_cache - .location - .join("http/localhost_PORT4546/cli/tests/subdir/redirects/redirect1.js"); + let redirect_path = fetcher.http_cache.get_cache_filename(&redirect_url); let target_url = Url::parse( "http://localhost:4545/cli/tests/subdir/redirects/redirect1.js", ) .unwrap(); - let target_path = fetcher - .deps_cache - .location - .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js"); + let target_path = fetcher.http_cache.get_cache_filename(&target_url); // Test double redirects and headers recording let result = fetcher @@ -1294,28 +999,22 @@ mod tests { .await; assert!(result.is_ok()); let mod_meta = result.unwrap(); - assert!(fs::read_to_string(&double_redirect_path).is_err()); - assert!(fs::read_to_string(&redirect_path).is_err()); + assert_eq!(fs::read_to_string(&double_redirect_path).unwrap(), ""); + assert_eq!(fs::read_to_string(&redirect_path).unwrap(), ""); - let double_redirect_headers = - fetcher.get_source_code_headers(&double_redirect_url); - assert_eq!( - double_redirect_headers.redirect_to.unwrap(), - redirect_url.to_string() - ); - let redirect_headers = fetcher.get_source_code_headers(&redirect_url); - assert_eq!( - redirect_headers.redirect_to.unwrap(), - target_url.to_string() - ); + let (_, headers) = fetcher.http_cache.get(&double_redirect_url).unwrap(); + assert_eq!(headers.get("location").unwrap(), &redirect_url.to_string()); + + let (_, headers) = fetcher.http_cache.get(&redirect_url).unwrap(); + assert_eq!(headers.get("location").unwrap(), &target_url.to_string()); // The target of redirection is downloaded instead. assert_eq!( fs::read_to_string(&target_path).unwrap(), "export const redirect = 1;\n" ); - let redirect_target_headers = fetcher.get_source_code_headers(&target_url); - assert!(redirect_target_headers.redirect_to.is_none()); + let (_, headers) = fetcher.http_cache.get(&target_url).unwrap(); + assert!(headers.get("location").is_none()); // Examine the meta result. assert_eq!(mod_meta.url, target_url); @@ -1338,10 +1037,7 @@ mod tests { ) .unwrap(); - let target_path = fetcher - .deps_cache - .location - .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js"); + let target_path = fetcher.http_cache.get_cache_filename(&redirect_url); let target_path_ = target_path.clone(); // Test that redirect target is not downloaded twice for different redirect source. @@ -1459,12 +1155,10 @@ mod tests { let module_url = Url::parse("http://127.0.0.1:4545/cli/tests/subdir/mt_video_mp2t.t3.ts") .unwrap(); - let headers_file_name = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&module_url, "headers.json"), - ); - + let headers_file_name = fetcher + .http_cache + .get_cache_filename(&module_url) + .with_extension("headers.json"); let result = fetcher .fetch_remote_source_async(&module_url, false, false, 10) .await; @@ -1472,15 +1166,10 @@ mod tests { let r = result.unwrap(); assert_eq!(r.source_code, b"export const loaded = true;\n"); assert_eq!(&(r.media_type), &msg::MediaType::TypeScript); - // matching ext, no .headers.json file created - assert!(fs::read_to_string(&headers_file_name).is_err()); // Modify .headers.json, make sure read from local - let _ = fetcher.save_source_code_headers( - &module_url, - Some("text/javascript".to_owned()), - None, - None, - None, + let _ = fs::write( + &headers_file_name, + "{ \"content-type\": \"text/javascript\" }", ); let result2 = fetcher.fetch_cached_remote_source(&module_url); assert!(result2.is_ok()); @@ -1500,11 +1189,6 @@ mod tests { let module_url = Url::parse("http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts") .unwrap(); - let headers_file_name = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&module_url, "headers.json"), - ); let result = fetcher .fetch_remote_source_async(&module_url, false, false, 10) @@ -1513,16 +1197,14 @@ mod tests { let r = result.unwrap(); assert_eq!(r.source_code, b"export const loaded = true;\n"); assert_eq!(&(r.media_type), &msg::MediaType::TypeScript); - // matching ext, no .headers.json file created - assert!(fs::read_to_string(&headers_file_name).is_err()); - // Modify .headers.json, make sure read from local - let _ = fetcher.save_source_code_headers( - &module_url, - Some("text/javascript".to_owned()), - None, - None, - None, + let headers_file_name = fetcher + .http_cache + .get_cache_filename(&module_url) + .with_extension("headers.json"); + let _ = fs::write( + &headers_file_name, + "{ \"content-type\": \"text/javascript\" }", ); let result2 = fetcher.fetch_cached_remote_source(&module_url); assert!(result2.is_ok()); @@ -1540,7 +1222,6 @@ mod tests { let (_temp_dir, fetcher) = test_setup(); let fetcher_1 = fetcher.clone(); let fetcher_2 = fetcher.clone(); - let fetcher_3 = fetcher.clone(); let module_url = Url::parse("http://localhost:4545/cli/tests/subdir/no_ext").unwrap(); let module_url_2 = @@ -1559,14 +1240,8 @@ mod tests { let r = result.unwrap(); assert_eq!(r.source_code, b"export const loaded = true;\n"); assert_eq!(&(r.media_type), &msg::MediaType::TypeScript); - // no ext, should create .headers.json file - assert_eq!( - fetcher_1 - .get_source_code_headers(&module_url) - .mime_type - .unwrap(), - "text/typescript" - ); + let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); + assert_eq!(headers.get("content-type").unwrap(), "text/typescript"); let result = fetcher_1 .fetch_remote_source_async(&module_url_2, false, false, 10) .await; @@ -1574,14 +1249,9 @@ mod tests { let r2 = result.unwrap(); assert_eq!(r2.source_code, b"export const loaded = true;\n"); assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript); - // mismatch ext, should create .headers.json file - assert_eq!( - fetcher_2 - .get_source_code_headers(&module_url_2_) - .mime_type - .unwrap(), - "text/javascript" - ); + let (_, headers) = fetcher.http_cache.get(&module_url_2_).unwrap(); + assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); + // test unknown extension let result = fetcher_2 .fetch_remote_source_async(&module_url_3, false, false, 10) @@ -1590,14 +1260,8 @@ mod tests { let r3 = result.unwrap(); assert_eq!(r3.source_code, b"export const loaded = true;\n"); assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript); - // unknown ext, should create .headers.json file - assert_eq!( - fetcher_3 - .get_source_code_headers(&module_url_3_) - .mime_type - .unwrap(), - "text/typescript" - ); + let (_, headers) = fetcher.http_cache.get(&module_url_3_).unwrap(); + assert_eq!(headers.get("content-type").unwrap(), "text/typescript"); drop(http_server_guard); } @@ -1904,23 +1568,21 @@ mod tests { assert_eq!(source.source_code, b"console.log('etag')"); assert_eq!(&(source.media_type), &msg::MediaType::TypeScript); - let headers = fetcher.get_source_code_headers(&module_url); - assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string())); + let (_, headers) = fetcher.http_cache.get(&module_url).unwrap(); + assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); - let header_path = fetcher.deps_cache.location.join( - fetcher - .deps_cache - .get_cache_filename_with_extension(&module_url, "headers.json"), - ); + let header_path = fetcher + .http_cache + .get_cache_filename(&module_url) + .with_extension("headers.json"); let modified1 = header_path.metadata().unwrap().modified().unwrap(); // Forcibly change the contents of the cache file and request // it again with the cache parameters turned off. // If the fetched content changes, the cached content is used. - fetcher - .save_source_code(&module_url, b"changed content") - .unwrap(); + let file_name = fetcher.http_cache.get_cache_filename(&module_url); + let _ = fs::write(&file_name, "changed content"); let cached_source = fetcher .fetch_remote_source_async(&module_url, false, false, 1) .await diff --git a/cli/global_state.rs b/cli/global_state.rs index 1d7a3a40f8..9880f18a2f 100644 --- a/cli/global_state.rs +++ b/cli/global_state.rs @@ -9,6 +9,7 @@ use crate::deno_dir; use crate::deno_error::permission_denied; use crate::file_fetcher::SourceFileFetcher; use crate::flags; +use crate::http_cache; use crate::lockfile::Lockfile; use crate::msg; use crate::permissions::DenoPermissions; @@ -58,9 +59,11 @@ impl GlobalState { pub fn new(flags: flags::DenoFlags) -> Result { let custom_root = env::var("DENO_DIR").map(String::into).ok(); let dir = deno_dir::DenoDir::new(custom_root)?; + let deps_cache_location = dir.root.join("deps"); + let http_cache = http_cache::HttpCache::new(&deps_cache_location)?; let file_fetcher = SourceFileFetcher::new( - dir.deps_cache.clone(), + http_cache, !flags.reload, flags.cache_blacklist.clone(), flags.no_remote, diff --git a/cli/http_cache.rs b/cli/http_cache.rs new file mode 100644 index 0000000000..4a604efd21 --- /dev/null +++ b/cli/http_cache.rs @@ -0,0 +1,195 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +/// This module is meant to eventually implement HTTP cache +/// as defined in RFC 7234 (https://tools.ietf.org/html/rfc7234). +/// Currently it's a very simplified version to fulfill Deno needs +/// at hand. +use crate::fs as deno_fs; +use crate::http_util::HeadersMap; +use deno_core::ErrBox; +use std::fs; +use std::fs::File; +use std::path::Path; +use std::path::PathBuf; +use url::Url; + +/// Turn base of url (scheme, hostname, port) into a valid filename. +/// This method replaces port part with a special string token (because +/// ":" cannot be used in filename on some platforms). +pub fn base_url_to_filename(url: &Url) -> PathBuf { + let mut out = PathBuf::new(); + + let scheme = url.scheme(); + out.push(scheme); + + match scheme { + "http" | "https" => { + let host = url.host_str().unwrap(); + let host_port = match url.port() { + Some(port) => format!("{}_PORT{}", host, port), + None => host.to_string(), + }; + out.push(host_port); + } + scheme => { + unimplemented!( + "Don't know how to create cache name for scheme: {}", + scheme + ); + } + }; + + out +} + +/// Turn provided `url` into a hashed filename. +/// URLs can contain a lot of characters that cannot be used +/// in filenames (like "?", "#", ":"), so in order to cache +/// them properly they are deterministically hashed into ASCII +/// strings. +/// +/// NOTE: this method is `pub` because it's used in integration_tests +pub fn url_to_filename(url: &Url) -> PathBuf { + let mut cache_filename = base_url_to_filename(url); + + let mut rest_str = url.path().to_string(); + if let Some(query) = url.query() { + rest_str.push_str("?"); + rest_str.push_str(query); + } + // NOTE: fragment is omitted on purpose - it's not taken into + // account when caching - it denotes parts of webpage, which + // in case of static resources doesn't make much sense + let hashed_filename = crate::checksum::gen(vec![rest_str.as_bytes()]); + cache_filename.push(hashed_filename); + cache_filename +} + +#[derive(Clone)] +pub struct HttpCache { + pub location: PathBuf, +} + +impl HttpCache { + /// Returns error if unable to create directory + /// at specified location. + pub fn new(location: &Path) -> Result { + fs::create_dir_all(&location)?; + Ok(Self { + location: location.to_owned(), + }) + } + + pub(crate) fn get_cache_filename(&self, url: &Url) -> PathBuf { + self.location.join(url_to_filename(url)) + } + + // TODO(bartlomieju): this method should check headers file + // and validate against ETAG/Last-modified-as headers. + // ETAG check is currently done in `cli/file_fetcher.rs`. + pub fn get(&self, url: &Url) -> Result<(File, HeadersMap), ErrBox> { + let cache_filename = self.location.join(url_to_filename(url)); + let headers_filename = cache_filename.with_extension("headers.json"); + let file = File::open(cache_filename)?; + let headers_json = fs::read_to_string(headers_filename)?; + let headers_map: HeadersMap = serde_json::from_str(&headers_json)?; + Ok((file, headers_map)) + } + + pub fn set( + &self, + url: &Url, + headers_map: HeadersMap, + content: &[u8], + ) -> Result<(), ErrBox> { + let cache_filename = self.location.join(url_to_filename(url)); + let headers_filename = cache_filename.with_extension("headers.json"); + // Create parent directory + let parent_filename = cache_filename + .parent() + .expect("Cache filename should have a parent dir"); + fs::create_dir_all(parent_filename)?; + // Cache content + deno_fs::write_file(&cache_filename, content, 0o666)?; + let serialized_headers = serde_json::to_string(&headers_map)?; + // Cache headers + deno_fs::write_file(&headers_filename, serialized_headers, 0o666)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + use std::io::Read; + use tempfile::TempDir; + + #[test] + fn test_create_cache() { + let dir = TempDir::new().unwrap(); + let mut cache_path = dir.path().to_owned(); + cache_path.push("foobar"); + let r = HttpCache::new(&cache_path); + assert!(r.is_ok()); + assert!(cache_path.is_dir()); + } + + #[test] + fn test_get_set() { + let dir = TempDir::new().unwrap(); + let cache = HttpCache::new(dir.path()).unwrap(); + let url = Url::parse("https://deno.land/x/welcome.ts").unwrap(); + let mut headers = HashMap::new(); + headers.insert( + "content-type".to_string(), + "application/javascript".to_string(), + ); + headers.insert("etag".to_string(), "as5625rqdsfb".to_string()); + let content = b"Hello world"; + let r = cache.set(&url, headers, content); + eprintln!("result {:?}", r); + assert!(r.is_ok()); + let r = cache.get(&url); + assert!(r.is_ok()); + let (mut file, headers) = r.unwrap(); + let mut content = String::new(); + file.read_to_string(&mut content).unwrap(); + assert_eq!(content, "Hello world"); + assert_eq!( + headers.get("content-type").unwrap(), + "application/javascript" + ); + assert_eq!(headers.get("etag").unwrap(), "as5625rqdsfb"); + assert_eq!(headers.get("foobar"), None); + drop(dir); + } + + #[test] + fn test_url_to_filename() { + let test_cases = [ + ("https://deno.land/x/foo.ts", "https/deno.land/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8"), + ( + "https://deno.land:8080/x/foo.ts", + "https/deno.land_PORT8080/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8", + ), + ("https://deno.land/", "https/deno.land/8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1"), + ( + "https://deno.land/?asdf=qwer", + "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", + ), + // should be the same as case above, fragment (#qwer) is ignored + // when hashing + ( + "https://deno.land/?asdf=qwer#qwer", + "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", + ), + ]; + + for (url, expected) in test_cases.iter() { + let u = Url::parse(url).unwrap(); + let p = url_to_filename(&u); + assert_eq!(p, PathBuf::from(expected)); + } + } +} diff --git a/cli/http_util.rs b/cli/http_util.rs index 0140d014a3..7da2c7dbdf 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -12,8 +12,6 @@ use reqwest::header::HeaderMap; use reqwest::header::HeaderValue; use reqwest::header::ACCEPT_ENCODING; use reqwest::header::CONTENT_ENCODING; -use reqwest::header::CONTENT_TYPE; -use reqwest::header::ETAG; use reqwest::header::IF_NONE_MATCH; use reqwest::header::LOCATION; use reqwest::header::USER_AGENT; @@ -22,6 +20,7 @@ use reqwest::Client; use reqwest::Response; use reqwest::StatusCode; use std::cmp::min; +use std::collections::HashMap; use std::fs::File; use std::future::Future; use std::io; @@ -86,19 +85,13 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { } } -#[derive(Debug, PartialEq)] -pub struct ResultPayload { - pub body: Vec, - pub content_type: Option, - pub etag: Option, - pub x_typescript_types: Option, -} +pub type HeadersMap = HashMap; #[derive(Debug, PartialEq)] pub enum FetchOnceResult { - Code(ResultPayload), + Code(Vec, HeadersMap), NotModified, - Redirect(Url), + Redirect(Url, HeadersMap), } /// Asynchronously fetches the given HTTP URL one pass only. @@ -128,6 +121,19 @@ pub fn fetch_once( return Ok(FetchOnceResult::NotModified); } + let mut headers_: HashMap = HashMap::new(); + let headers = response.headers(); + for key in headers.keys() { + let key_str = key.to_string(); + let values = headers.get_all(key); + let values_str = values + .iter() + .map(|e| e.to_str().unwrap().to_string()) + .collect::>() + .join(","); + headers_.insert(key_str, values_str); + } + if response.status().is_redirection() { let location_string = response .headers() @@ -138,7 +144,7 @@ pub fn fetch_once( debug!("Redirecting to {:?}...", &location_string); let new_url = resolve_url_from_location(&url, location_string); - return Ok(FetchOnceResult::Redirect(new_url)); + return Ok(FetchOnceResult::Redirect(new_url, headers_)); } if response.status().is_client_error() @@ -151,31 +157,11 @@ pub fn fetch_once( return Err(err.into()); } - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(|content_type| content_type.to_str().unwrap().to_owned()); - - let etag = response - .headers() - .get(ETAG) - .map(|etag| etag.to_str().unwrap().to_owned()); - let content_encoding = response .headers() .get(CONTENT_ENCODING) .map(|content_encoding| content_encoding.to_str().unwrap().to_owned()); - const X_TYPESCRIPT_TYPES: &str = "X-TypeScript-Types"; - - let x_typescript_types = - response - .headers() - .get(X_TYPESCRIPT_TYPES) - .map(|x_typescript_types| { - x_typescript_types.to_str().unwrap().to_owned() - }); - let body; if let Some(content_encoding) = content_encoding { body = match content_encoding { @@ -192,12 +178,7 @@ pub fn fetch_once( body = response.bytes().await?.to_vec(); } - return Ok(FetchOnceResult::Code(ResultPayload { - body, - content_type, - etag, - x_typescript_types, - })); + return Ok(FetchOnceResult::Code(body, headers_)); }; fut.boxed() @@ -291,11 +272,11 @@ mod tests { Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap(); let client = create_http_client(None).unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { - assert!(!payload.body.is_empty()); - assert_eq!(payload.content_type, Some("application/json".to_string())); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert!(!body.is_empty()); + assert_eq!(headers.get("content-type").unwrap(), "application/json"); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -312,17 +293,14 @@ mod tests { .unwrap(); let client = create_http_client(None).unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( - String::from_utf8(payload.body).unwrap(), - "console.log('gzip')" + headers.get("content-type").unwrap(), + "application/javascript" ); - assert_eq!( - payload.content_type, - Some("application/javascript".to_string()) - ); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -335,18 +313,14 @@ mod tests { let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); let client = create_http_client(None).unwrap(); let result = fetch_once(client.clone(), &url, None).await; - if let Ok(FetchOnceResult::Code(ResultPayload { - body, - content_type, - etag, - x_typescript_types, - })) = result - { + if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); - assert_eq!(content_type, Some("application/typescript".to_string())); - assert_eq!(etag, Some("33a64df551425fcc55e".to_string())); - assert_eq!(x_typescript_types, None); + assert_eq!( + headers.get("content-type").unwrap(), + "application/typescript" + ); + assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); } else { panic!(); } @@ -368,18 +342,15 @@ mod tests { .unwrap(); let client = create_http_client(None).unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { - assert!(!payload.body.is_empty()); + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!( - String::from_utf8(payload.body).unwrap(), - "console.log('brotli');" + headers.get("content-type").unwrap(), + "application/javascript" ); - assert_eq!( - payload.content_type, - Some("application/javascript".to_string()) - ); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -397,7 +368,7 @@ mod tests { Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap(); let client = create_http_client(None).unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Redirect(url)) = result { + if let Ok(FetchOnceResult::Redirect(url, _)) = result { assert_eq!(url, target_url); } else { panic!(); @@ -459,11 +430,11 @@ mod tests { .unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { - assert!(!payload.body.is_empty()); - assert_eq!(payload.content_type, Some("application/json".to_string())); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert!(!body.is_empty()); + assert_eq!(headers.get("content-type").unwrap(), "application/json"); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -486,17 +457,14 @@ mod tests { ))) .unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( - String::from_utf8(payload.body).unwrap(), - "console.log('gzip')" + headers.get("content-type").unwrap(), + "application/javascript" ); - assert_eq!( - payload.content_type, - Some("application/javascript".to_string()) - ); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -515,18 +483,15 @@ mod tests { ))) .unwrap(); let result = fetch_once(client.clone(), &url, None).await; - if let Ok(FetchOnceResult::Code(ResultPayload { - body, - content_type, - etag, - x_typescript_types, - })) = result - { + if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); - assert_eq!(content_type, Some("application/typescript".to_string())); - assert_eq!(etag, Some("33a64df551425fcc55e".to_string())); - assert_eq!(x_typescript_types, None); + assert_eq!( + headers.get("content-type").unwrap(), + "application/typescript" + ); + assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e"); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } @@ -554,18 +519,15 @@ mod tests { ))) .unwrap(); let result = fetch_once(client, &url, None).await; - if let Ok(FetchOnceResult::Code(payload)) = result { - assert!(!payload.body.is_empty()); + if let Ok(FetchOnceResult::Code(body, headers)) = result { + assert!(!body.is_empty()); + assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!( - String::from_utf8(payload.body).unwrap(), - "console.log('brotli');" + headers.get("content-type").unwrap(), + "application/javascript" ); - assert_eq!( - payload.content_type, - Some("application/javascript".to_string()) - ); - assert_eq!(payload.etag, None); - assert_eq!(payload.x_typescript_types, None); + assert_eq!(headers.get("etag"), None); + assert_eq!(headers.get("x-typescript-types"), None); } else { panic!(); } diff --git a/cli/lib.rs b/cli/lib.rs index 7737d1741c..3edcad7ebd 100644 --- a/cli/lib.rs +++ b/cli/lib.rs @@ -33,6 +33,7 @@ pub mod fmt_errors; mod fs; mod global_state; mod global_timer; +pub mod http_cache; mod http_util; mod import_map; pub mod installer; @@ -132,7 +133,7 @@ fn print_cache_info(state: &GlobalState) { println!( "{} {:?}", colors::bold("Remote modules cache:".to_string()), - state.dir.deps_cache.location + state.file_fetcher.http_cache.location ); println!( "{} {:?}", diff --git a/cli/tests/022_info_flag_script.out b/cli/tests/022_info_flag_script.out index 48eef73656..14580839e6 100644 --- a/cli/tests/022_info_flag_script.out +++ b/cli/tests/022_info_flag_script.out @@ -1,4 +1,4 @@ -local: [WILDCARD]019_media_types.ts +local: [WILDCARD]http[WILDCARD]127.0.0.1_PORT4545[WILDCARD] type: TypeScript compiled: [WILDCARD].js map: [WILDCARD].js.map diff --git a/cli/tests/049_info_flag_script_jsx.out b/cli/tests/049_info_flag_script_jsx.out index 48f5efa9e1..135a830adc 100644 --- a/cli/tests/049_info_flag_script_jsx.out +++ b/cli/tests/049_info_flag_script_jsx.out @@ -1,4 +1,4 @@ -local: [WILDCARD]048_media_types_jsx.ts +local: [WILDCARD]http[WILDCARD]127.0.0.1_PORT4545[WILDCARD] type: TypeScript compiled: [WILDCARD]048_media_types_jsx.ts.js map: [WILDCARD]048_media_types_jsx.ts.js.map diff --git a/cli/tests/cafile_info.ts.out b/cli/tests/cafile_info.ts.out index 443b92eea6..c509907051 100644 --- a/cli/tests/cafile_info.ts.out +++ b/cli/tests/cafile_info.ts.out @@ -1,4 +1,4 @@ -local: [WILDCARD]cafile_info.ts +local: [WILDCARD]https[WILDCARD]localhost_PORT5545[WILDCARD] type: TypeScript compiled: [WILDCARD].js map: [WILDCARD].js.map diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index ebb0349c55..bfe5794f53 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -23,20 +23,23 @@ fn deno_dir_test() { #[test] fn fetch_test() { + use deno::http_cache::url_to_filename; pub use deno::test_util::*; use std::process::Command; use tempfile::TempDir; + use url::Url; let g = util::http_server(); let deno_dir = TempDir::new().expect("tempdir fail"); - let t = util::root_path().join("cli/tests/006_url_imports.ts"); + let module_url = + Url::parse("http://localhost:4545/cli/tests/006_url_imports.ts").unwrap(); let output = Command::new(deno_exe_path()) .env("DENO_DIR", deno_dir.path()) .current_dir(util::root_path()) .arg("fetch") - .arg(t) + .arg(module_url.to_string()) .output() .expect("Failed to spawn script"); @@ -48,7 +51,8 @@ fn fetch_test() { let expected_path = deno_dir .path() - .join("deps/http/localhost_PORT4545/cli/tests/subdir/mod2.ts"); + .join("deps") + .join(url_to_filename(&module_url)); assert_eq!(expected_path.exists(), true); drop(g); @@ -966,14 +970,18 @@ itest!(cafile_info { #[test] fn cafile_fetch() { + use deno::http_cache::url_to_filename; pub use deno::test_util::*; use std::process::Command; use tempfile::TempDir; + use url::Url; let g = util::http_server(); let deno_dir = TempDir::new().expect("tempdir fail"); - let t = util::root_path().join("cli/tests/cafile_url_imports.ts"); + let module_url = + Url::parse("http://localhost:4545/cli/tests/cafile_url_imports.ts") + .unwrap(); let cafile = util::root_path().join("cli/tests/tls/RootCA.pem"); let output = Command::new(deno_exe_path()) .env("DENO_DIR", deno_dir.path()) @@ -981,7 +989,7 @@ fn cafile_fetch() { .arg("fetch") .arg("--cert") .arg(cafile) - .arg(t) + .arg(module_url.to_string()) .output() .expect("Failed to spawn script"); @@ -993,7 +1001,8 @@ fn cafile_fetch() { let expected_path = deno_dir .path() - .join("deps/https/localhost_PORT5545/cli/tests/subdir/mod2.ts"); + .join("deps") + .join(url_to_filename(&module_url)); assert_eq!(expected_path.exists(), true); drop(g);