diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index e16b808d97..b9a91d41a0 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -3,13 +3,12 @@ use crate::auth_tokens::AuthTokens; use crate::colors; use crate::http_cache::HttpCache; -use crate::http_util::fetch_once; use crate::http_util::CacheSemantics; use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceResult; +use crate::http_util::HttpClient; use crate::progress_bar::ProgressBar; use crate::text_encoding; -use crate::version::get_user_agent; use data_url::DataUrl; use deno_ast::MediaType; @@ -22,8 +21,6 @@ use deno_core::futures; use deno_core::futures::future::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; -use deno_runtime::deno_fetch::create_http_client; -use deno_runtime::deno_fetch::reqwest; use deno_runtime::deno_tls::rustls; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::rustls_native_certs::load_native_certs; @@ -333,7 +330,7 @@ pub struct FileFetcher { cache: FileCache, cache_setting: CacheSetting, pub http_cache: HttpCache, - http_client: reqwest::Client, + http_client: HttpClient, blob_store: BlobStore, download_log_level: log::Level, progress_bar: Option, @@ -344,9 +341,8 @@ impl FileFetcher { http_cache: HttpCache, cache_setting: CacheSetting, allow_remote: bool, - root_cert_store: Option, + http_client: HttpClient, blob_store: BlobStore, - unsafely_ignore_certificate_errors: Option>, progress_bar: Option, ) -> Result { Ok(Self { @@ -355,14 +351,7 @@ impl FileFetcher { cache: Default::default(), cache_setting, http_cache, - http_client: create_http_client( - get_user_agent(), - root_cert_store, - vec![], - None, - unsafely_ignore_certificate_errors, - None, - )?, + http_client, blob_store, download_log_level: log::Level::Info, progress_bar, @@ -628,14 +617,14 @@ impl FileFetcher { let file_fetcher = self.clone(); // A single pass of fetch either yields code or yields a redirect. async move { - match fetch_once(FetchOnceArgs { - client, - url: specifier.clone(), - maybe_accept: maybe_accept.clone(), - maybe_etag, - maybe_auth_token, - }) - .await? + match client + .fetch_once(FetchOnceArgs { + url: specifier.clone(), + maybe_accept: maybe_accept.clone(), + maybe_etag, + maybe_auth_token, + }) + .await? { FetchOnceResult::NotModified => { let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap(); @@ -765,6 +754,8 @@ impl FileFetcher { #[cfg(test)] mod tests { + use crate::http_util::HttpClient; + use super::*; use deno_core::error::get_custom_error_class; use deno_core::resolve_url; @@ -793,10 +784,9 @@ mod tests { HttpCache::new(&location), cache_setting, true, - None, + HttpClient::new(None, None).unwrap(), blob_store.clone(), None, - None, ) .unwrap(); (file_fetcher, temp_dir, blob_store) @@ -1232,10 +1222,9 @@ mod tests { HttpCache::new(&location), CacheSetting::ReloadAll, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let result = file_fetcher @@ -1259,10 +1248,9 @@ mod tests { HttpCache::new(&location), CacheSetting::Use, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let specifier = @@ -1287,10 +1275,9 @@ mod tests { HttpCache::new(&location), CacheSetting::Use, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let result = file_fetcher_02 @@ -1431,10 +1418,9 @@ mod tests { HttpCache::new(&location), CacheSetting::Use, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let specifier = @@ -1461,10 +1447,9 @@ mod tests { HttpCache::new(&location), CacheSetting::Use, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let result = file_fetcher_02 @@ -1562,10 +1547,9 @@ mod tests { HttpCache::new(&location), CacheSetting::Use, false, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let specifier = @@ -1589,20 +1573,18 @@ mod tests { HttpCache::new(&location), CacheSetting::Only, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let file_fetcher_02 = FileFetcher::new( HttpCache::new(&location), CacheSetting::Use, true, - None, + HttpClient::new(None, None).unwrap(), BlobStore::default(), None, - None, ) .unwrap(); let specifier = diff --git a/cli/http_cache.rs b/cli/http_cache.rs index f70ccc715b..6cda9f2794 100644 --- a/cli/http_cache.rs +++ b/cli/http_cache.rs @@ -73,11 +73,6 @@ pub fn url_to_filename(url: &Url) -> Option { Some(cache_filename) } -#[derive(Debug, Clone, Default)] -pub struct HttpCache { - pub location: PathBuf, -} - #[derive(Serialize, Deserialize)] pub struct Metadata { pub headers: HeadersMap, @@ -107,6 +102,11 @@ impl Metadata { } } +#[derive(Debug, Clone, Default)] +pub struct HttpCache { + pub location: PathBuf, +} + impl HttpCache { /// Returns a new instance. /// diff --git a/cli/http_util.rs b/cli/http_util.rs index 54d641befa..5cb843a426 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -1,5 +1,6 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use crate::auth_tokens::AuthToken; +use crate::version::get_user_agent; use cache_control::Cachability; use cache_control::CacheControl; @@ -8,13 +9,15 @@ use deno_core::error::custom_error; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::url::Url; +use deno_runtime::deno_fetch::create_http_client; +use deno_runtime::deno_fetch::reqwest; use deno_runtime::deno_fetch::reqwest::header::HeaderValue; use deno_runtime::deno_fetch::reqwest::header::ACCEPT; use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH; use deno_runtime::deno_fetch::reqwest::header::LOCATION; -use deno_runtime::deno_fetch::reqwest::Client; use deno_runtime::deno_fetch::reqwest::StatusCode; +use deno_runtime::deno_tls::rustls::RootCertStore; use log::debug; use std::collections::HashMap; use std::time::Duration; @@ -208,97 +211,125 @@ pub enum FetchOnceResult { #[derive(Debug)] pub struct FetchOnceArgs { - pub client: Client, pub url: Url, pub maybe_accept: Option, pub maybe_etag: Option, pub maybe_auth_token: Option, } -/// Asynchronously fetches the given HTTP URL one pass only. -/// If no redirect is present and no error occurs, -/// yields Code(ResultPayload). -/// If redirect occurs, does not follow and -/// yields Redirect(url). -pub async fn fetch_once( - args: FetchOnceArgs, -) -> Result { - let mut request = args.client.get(args.url.clone()); +#[derive(Debug, Clone)] +pub struct HttpClient(reqwest::Client); - if let Some(etag) = args.maybe_etag { - let if_none_match_val = HeaderValue::from_str(&etag)?; - request = request.header(IF_NONE_MATCH, if_none_match_val); - } - if let Some(auth_token) = args.maybe_auth_token { - let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; - request = request.header(AUTHORIZATION, authorization_val); - } - if let Some(accept) = args.maybe_accept { - let accepts_val = HeaderValue::from_str(&accept)?; - request = request.header(ACCEPT, accepts_val); - } - let response = request.send().await?; - - if response.status() == StatusCode::NOT_MODIFIED { - return Ok(FetchOnceResult::NotModified); +impl HttpClient { + pub fn new( + root_cert_store: Option, + unsafely_ignore_certificate_errors: Option>, + ) -> Result { + Ok(HttpClient::from_client(create_http_client( + get_user_agent(), + root_cert_store, + vec![], + None, + unsafely_ignore_certificate_errors, + None, + )?)) } - let mut headers_: HashMap = HashMap::new(); - let headers = response.headers(); - - if let Some(warning) = headers.get("X-Deno-Warning") { - eprintln!( - "{} {}", - crate::colors::yellow("Warning"), - warning.to_str().unwrap() - ); + pub fn from_client(client: reqwest::Client) -> Self { + Self(client) } - for key in headers.keys() { - let key_str = key.to_string(); - let values = headers.get_all(key); - let values_str = values - .iter() - .map(|e| e.to_str().unwrap().to_string()) - .collect::>() - .join(","); - headers_.insert(key_str, values_str); + pub fn get(&self, url: U) -> reqwest::RequestBuilder { + self.0.get(url) } - if response.status().is_redirection() { - if let Some(location) = response.headers().get(LOCATION) { - let location_string = location.to_str().unwrap(); - debug!("Redirecting to {:?}...", &location_string); - let new_url = resolve_url_from_location(&args.url, location_string); - return Ok(FetchOnceResult::Redirect(new_url, headers_)); - } else { - return Err(generic_error(format!( - "Redirection from '{}' did not provide location header", - args.url - ))); + /// Asynchronously fetches the given HTTP URL one pass only. + /// If no redirect is present and no error occurs, + /// yields Code(ResultPayload). + /// If redirect occurs, does not follow and + /// yields Redirect(url). + pub async fn fetch_once( + &self, + args: FetchOnceArgs, + ) -> Result { + let mut request = self.get(args.url.clone()); + + if let Some(etag) = args.maybe_etag { + let if_none_match_val = HeaderValue::from_str(&etag)?; + request = request.header(IF_NONE_MATCH, if_none_match_val); } + if let Some(auth_token) = args.maybe_auth_token { + let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; + request = request.header(AUTHORIZATION, authorization_val); + } + if let Some(accept) = args.maybe_accept { + let accepts_val = HeaderValue::from_str(&accept)?; + request = request.header(ACCEPT, accepts_val); + } + let response = request.send().await?; + + if response.status() == StatusCode::NOT_MODIFIED { + return Ok(FetchOnceResult::NotModified); + } + + let mut result_headers = HashMap::new(); + let response_headers = response.headers(); + + if let Some(warning) = response_headers.get("X-Deno-Warning") { + eprintln!( + "{} {}", + crate::colors::yellow("Warning"), + warning.to_str().unwrap() + ); + } + + for key in response_headers.keys() { + let key_str = key.to_string(); + let values = response_headers.get_all(key); + let values_str = values + .iter() + .map(|e| e.to_str().unwrap().to_string()) + .collect::>() + .join(","); + result_headers.insert(key_str, values_str); + } + + if response.status().is_redirection() { + if let Some(location) = response.headers().get(LOCATION) { + let location_string = location.to_str().unwrap(); + debug!("Redirecting to {:?}...", &location_string); + let new_url = resolve_url_from_location(&args.url, location_string); + return Ok(FetchOnceResult::Redirect(new_url, result_headers)); + } else { + return Err(generic_error(format!( + "Redirection from '{}' did not provide location header", + args.url + ))); + } + } + + if response.status().is_client_error() + || response.status().is_server_error() + { + let err = if response.status() == StatusCode::NOT_FOUND { + custom_error( + "NotFound", + format!("Import '{}' failed, not found.", args.url), + ) + } else { + generic_error(format!( + "Import '{}' failed: {}", + args.url, + response.status() + )) + }; + return Err(err); + } + + let body = response.bytes().await?.to_vec(); + + Ok(FetchOnceResult::Code(body, result_headers)) } - - if response.status().is_client_error() || response.status().is_server_error() - { - let err = if response.status() == StatusCode::NOT_FOUND { - custom_error( - "NotFound", - format!("Import '{}' failed, not found.", args.url), - ) - } else { - generic_error(format!( - "Import '{}' failed: {}", - args.url, - response.status() - )) - }; - return Err(err); - } - - let body = response.bytes().await?.to_vec(); - - Ok(FetchOnceResult::Code(body, headers_)) } #[cfg(test)] @@ -308,16 +339,18 @@ mod tests { use deno_runtime::deno_fetch::create_http_client; use std::fs::read; - fn create_test_client() -> Client { - create_http_client( - "test_client".to_string(), - None, - vec![], - None, - None, - None, + fn create_test_client() -> HttpClient { + HttpClient::from_client( + create_http_client( + "test_client".to_string(), + None, + vec![], + None, + None, + None, + ) + .unwrap(), ) - .unwrap() } #[tokio::test] @@ -326,14 +359,14 @@ mod tests { // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(headers.get("content-type").unwrap(), "application/json"); @@ -351,14 +384,14 @@ mod tests { let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped") .unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( @@ -377,14 +410,14 @@ mod tests { let _http_server_guard = test_util::http_server(); let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client: client.clone(), - url: url.clone(), - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url: url.clone(), + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); @@ -397,14 +430,14 @@ mod tests { panic!(); } - let res = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: Some("33a64df551425fcc55e".to_string()), - maybe_auth_token: None, - }) - .await; + let res = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: Some("33a64df551425fcc55e".to_string()), + maybe_auth_token: None, + }) + .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); } @@ -415,14 +448,14 @@ mod tests { let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli") .unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); @@ -443,14 +476,14 @@ mod tests { // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: Some("application/json".to_string()), - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: Some("application/json".to_string()), + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, _)) = result { assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes()); } else { @@ -467,14 +500,14 @@ mod tests { let target_url = Url::parse("http://localhost:4545/assets/fixture.json").unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Redirect(url, _)) = result { assert_eq!(url, target_url); } else { @@ -526,29 +559,31 @@ mod tests { // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap(); - let client = create_http_client( - version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), + let client = HttpClient::from_client( + create_http_client( + version::get_user_agent(), + None, + vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + None, + None, + None, ) - .unwrap()], - None, - None, - None, - ) - .unwrap(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + .unwrap(), + ); + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(headers.get("content-type").unwrap(), "application/json"); @@ -564,24 +599,26 @@ mod tests { let _http_server_guard = test_util::http_server(); // Relies on external http server with a valid mozilla root CA cert. let url = Url::parse("https://deno.land").unwrap(); - let client = create_http_client( - version::get_user_agent(), - None, // This will load mozilla certs by default - vec![], - None, - None, - None, - ) - .unwrap(); + let client = HttpClient::from_client( + create_http_client( + version::get_user_agent(), + None, // This will load mozilla certs by default + vec![], + None, + None, + None, + ) + .unwrap(), + ); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; println!("{:?}", result); if let Ok(FetchOnceResult::Code(body, _headers)) = result { @@ -601,24 +638,20 @@ mod tests { let _http_server_guard = test_util::http_server(); // Relies on external http server with a valid mozilla root CA cert. let url = Url::parse("https://deno.land").unwrap(); - let client = create_http_client( - version::get_user_agent(), + let client = HttpClient::new( Some(RootCertStore::empty()), // no certs loaded at all - vec![], - None, - None, None, ) .unwrap(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(_body, _headers)) = result { // This test is expected to fail since to CA certs have been loaded @@ -633,29 +666,31 @@ mod tests { let url = Url::parse("https://localhost:5545/run/import_compression/gziped") .unwrap(); - let client = create_http_client( - version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), + let client = HttpClient::from_client( + create_http_client( + version::get_user_agent(), + None, + vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + None, + None, + None, ) - .unwrap()], - None, - None, - None, - ) - .unwrap(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + .unwrap(), + ); + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!( @@ -673,29 +708,31 @@ mod tests { async fn test_fetch_with_cafile_with_etag() { let _http_server_guard = test_util::http_server(); let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); - let client = create_http_client( - version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), + let client = HttpClient::from_client( + create_http_client( + version::get_user_agent(), + None, + vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + None, + None, + None, ) - .unwrap()], - None, - None, - None, - ) - .unwrap(); - let result = fetch_once(FetchOnceArgs { - client: client.clone(), - url: url.clone(), - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + .unwrap(), + ); + let result = client + .fetch_once(FetchOnceArgs { + url: url.clone(), + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); @@ -709,14 +746,14 @@ mod tests { panic!(); } - let res = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: Some("33a64df551425fcc55e".to_string()), - maybe_auth_token: None, - }) - .await; + let res = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: Some("33a64df551425fcc55e".to_string()), + maybe_auth_token: None, + }) + .await; assert_eq!(res.unwrap(), FetchOnceResult::NotModified); } @@ -727,29 +764,31 @@ mod tests { let url = Url::parse("https://localhost:5545/run/import_compression/brotli") .unwrap(); - let client = create_http_client( - version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), + let client = HttpClient::from_client( + create_http_client( + version::get_user_agent(), + None, + vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + None, + None, + None, ) - .unwrap()], - None, - None, - None, - ) - .unwrap(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + .unwrap(), + ); + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; if let Ok(FetchOnceResult::Code(body, headers)) = result { assert!(!body.is_empty()); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); @@ -770,14 +809,14 @@ mod tests { let url_str = "http://127.0.0.1:4545/bad_redirect"; let url = Url::parse(url_str).unwrap(); let client = create_test_client(); - let result = fetch_once(FetchOnceArgs { - client, - url, - maybe_accept: None, - maybe_etag: None, - maybe_auth_token: None, - }) - .await; + let result = client + .fetch_once(FetchOnceArgs { + url, + maybe_accept: None, + maybe_etag: None, + maybe_auth_token: None, + }) + .await; assert!(result.is_err()); let err = result.unwrap_err(); // Check that the error message contains the original URL diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index cf1fbba976..4595d21465 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -50,7 +50,6 @@ use super::parent_process_checker; use super::performance::Performance; use super::refactor; use super::registries::ModuleRegistry; -use super::registries::ModuleRegistryOptions; use super::testing; use super::text; use super::tsc; @@ -65,10 +64,13 @@ use crate::args::FmtConfig; use crate::args::LintConfig; use crate::args::TsConfig; use crate::deno_dir; +use crate::deno_dir::DenoDir; +use crate::file_fetcher::get_root_cert_store; use crate::file_fetcher::get_source_from_data_url; use crate::file_fetcher::CacheSetting; use crate::fs_util; use crate::graph_util::graph_valid; +use crate::http_util::HttpClient; use crate::npm::NpmCache; use crate::npm::NpmPackageResolver; use crate::npm::RealNpmRegistryApi; @@ -235,17 +237,43 @@ impl LanguageServer { } } +fn create_lsp_npm_resolver( + dir: &DenoDir, + http_client: HttpClient, +) -> NpmPackageResolver { + let registry_url = RealNpmRegistryApi::default_url(); + // Use an "only" cache setting in order to make the + // user do an explicit "cache" command and prevent + // the cache from being filled with lots of packages while + // the user is typing. + let cache_setting = CacheSetting::Only; + let progress_bar = ProgressBar::default(); + let npm_cache = NpmCache::from_deno_dir( + dir, + cache_setting.clone(), + http_client.clone(), + progress_bar.clone(), + ); + let api = RealNpmRegistryApi::new( + registry_url, + npm_cache.clone(), + cache_setting, + http_client, + progress_bar, + ); + NpmPackageResolver::new(npm_cache, api, false, None) +} + impl Inner { fn new(client: Client) -> Self { let maybe_custom_root = env::var("DENO_DIR").map(String::into).ok(); let dir = deno_dir::DenoDir::new(maybe_custom_root) .expect("could not access DENO_DIR"); let module_registries_location = dir.root.join(REGISTRIES_PATH); - let module_registries = ModuleRegistry::new( - &module_registries_location, - ModuleRegistryOptions::default(), - ) - .expect("could not create module registries"); + let http_client = HttpClient::new(None, None).unwrap(); + let module_registries = + ModuleRegistry::new(&module_registries_location, http_client.clone()) + .unwrap(); let location = dir.root.join(CACHE_PATH); let documents = Documents::new(&location); let cache_metadata = cache::CacheMetadata::new(&location); @@ -258,25 +286,7 @@ impl Inner { ts_server.clone(), ); let assets = Assets::new(ts_server.clone()); - let registry_url = RealNpmRegistryApi::default_url(); - // Use an "only" cache setting in order to make the - // user do an explicit "cache" command and prevent - // the cache from being filled with lots of packages while - // the user is typing. - let cache_setting = CacheSetting::Only; - let progress_bar = ProgressBar::default(); - let npm_cache = NpmCache::from_deno_dir( - &dir, - cache_setting.clone(), - progress_bar.clone(), - ); - let api = RealNpmRegistryApi::new( - registry_url, - npm_cache.clone(), - cache_setting, - progress_bar, - ); - let npm_resolver = NpmPackageResolver::new(npm_cache, api, false, None); + let npm_resolver = create_lsp_npm_resolver(&dir, http_client); Self { assets, @@ -498,36 +508,48 @@ impl Inner { None }; if self.maybe_cache_path != maybe_cache_path { - let maybe_custom_root = maybe_cache_path - .clone() - .or_else(|| env::var("DENO_DIR").map(String::into).ok()); - let dir = deno_dir::DenoDir::new(maybe_custom_root)?; - let module_registries_location = dir.root.join(REGISTRIES_PATH); - let workspace_settings = self.config.get_workspace_settings(); - let maybe_root_path = self - .config - .root_uri - .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); - self.module_registries = ModuleRegistry::new( - &module_registries_location, - ModuleRegistryOptions { - maybe_root_path, - maybe_ca_stores: workspace_settings.certificate_stores.clone(), - maybe_ca_file: workspace_settings.tls_certificate.clone(), - unsafely_ignore_certificate_errors: workspace_settings - .unsafely_ignore_certificate_errors, - }, - )?; - self.module_registries_location = module_registries_location; - let location = dir.root.join(CACHE_PATH); - self.documents.set_location(&location); - self.cache_metadata.set_location(&location); - self.maybe_cache_path = maybe_cache_path; + self.recreate_http_client_and_dependents(maybe_cache_path)?; } Ok(()) } + /// Recreates the http client and all dependent structs. + fn recreate_http_client_and_dependents( + &mut self, + new_cache_path: Option, + ) -> Result<(), AnyError> { + let maybe_custom_root = new_cache_path + .clone() + .or_else(|| env::var("DENO_DIR").map(String::into).ok()); + let dir = deno_dir::DenoDir::new(maybe_custom_root)?; + let workspace_settings = self.config.get_workspace_settings(); + let maybe_root_path = self + .config + .root_uri + .as_ref() + .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); + let root_cert_store = Some(get_root_cert_store( + maybe_root_path, + workspace_settings.certificate_stores.clone(), + workspace_settings.tls_certificate.clone(), + )?); + let client = HttpClient::new( + root_cert_store, + workspace_settings.unsafely_ignore_certificate_errors, + )?; + let module_registries_location = dir.root.join(REGISTRIES_PATH); + self.module_registries = + ModuleRegistry::new(&module_registries_location, client.clone())?; + self.module_registries_location = module_registries_location; + self.npm_resolver = create_lsp_npm_resolver(&dir, client); + // update the cache path + let location = dir.root.join(CACHE_PATH); + self.documents.set_location(&location); + self.cache_metadata.set_location(&location); + self.maybe_cache_path = new_cache_path; + Ok(()) + } + pub async fn update_import_map(&mut self) -> Result<(), AnyError> { let mark = self.performance.mark("update_import_map", None::<()>); @@ -627,23 +649,8 @@ impl Inner { async fn update_registries(&mut self) -> Result<(), AnyError> { let mark = self.performance.mark("update_registries", None::<()>); + self.recreate_http_client_and_dependents(self.maybe_cache_path.clone())?; let workspace_settings = self.config.get_workspace_settings(); - let maybe_root_path = self - .config - .root_uri - .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); - self.module_registries = ModuleRegistry::new( - &self.module_registries_location, - ModuleRegistryOptions { - maybe_root_path, - maybe_ca_stores: workspace_settings.certificate_stores.clone(), - maybe_ca_file: workspace_settings.tls_certificate.clone(), - unsafely_ignore_certificate_errors: workspace_settings - .unsafely_ignore_certificate_errors - .clone(), - }, - )?; for (registry, enabled) in workspace_settings.suggest.imports.hosts.iter() { if *enabled { lsp_log!("Enabling import suggestions for: {}", registry); diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 3da435666f..6fd48d8b9d 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -13,10 +13,10 @@ use super::path_to_regex::StringOrVec; use super::path_to_regex::Token; use crate::deno_dir; -use crate::file_fetcher::get_root_cert_store; use crate::file_fetcher::CacheSetting; use crate::file_fetcher::FileFetcher; use crate::http_cache::HttpCache; +use crate::http_util::HttpClient; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; @@ -36,7 +36,6 @@ use once_cell::sync::Lazy; use regex::Regex; use std::collections::HashMap; use std::path::Path; -use std::path::PathBuf; use tower_lsp::lsp_types as lsp; const CONFIG_PATH: &str = "/.well-known/deno-import-intellisense.json"; @@ -409,14 +408,6 @@ enum VariableItems { List(VariableItemsList), } -#[derive(Debug, Default)] -pub struct ModuleRegistryOptions { - pub maybe_root_path: Option, - pub maybe_ca_stores: Option>, - pub maybe_ca_file: Option, - pub unsafely_ignore_certificate_errors: Option>, -} - /// A structure which holds the information about currently configured module /// registries and can provide completion information for URLs that match /// one of the enabled registries. @@ -433,28 +424,23 @@ impl Default for ModuleRegistry { // custom root. let dir = deno_dir::DenoDir::new(None).unwrap(); let location = dir.root.join("registries"); - Self::new(&location, ModuleRegistryOptions::default()).unwrap() + let http_client = HttpClient::new(None, None).unwrap(); + Self::new(&location, http_client).unwrap() } } impl ModuleRegistry { pub fn new( location: &Path, - options: ModuleRegistryOptions, + http_client: HttpClient, ) -> Result { let http_cache = HttpCache::new(location); - let root_cert_store = Some(get_root_cert_store( - options.maybe_root_path, - options.maybe_ca_stores, - options.maybe_ca_file, - )?); let mut file_fetcher = FileFetcher::new( http_cache, CacheSetting::RespectHeaders, true, - root_cert_store, + http_client, BlobStore::default(), - options.unsafely_ignore_certificate_errors, None, )?; file_fetcher.set_download_log_level(super::logging::lsp_log_level()); @@ -1262,7 +1248,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let mut module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); module_registry .enable("http://localhost:4545/") .await @@ -1323,7 +1310,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let mut module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); module_registry .enable("http://localhost:4545/") .await @@ -1546,7 +1534,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let mut module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); module_registry .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-key-first.json") .await @@ -1616,7 +1605,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let mut module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); module_registry .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-complex.json") .await @@ -1667,7 +1657,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); let result = module_registry.check_origin("http://localhost:4545").await; assert!(result.is_ok()); } @@ -1678,7 +1669,8 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("registries"); let module_registry = - ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); + ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap()) + .unwrap(); let result = module_registry.check_origin("https://deno.com").await; assert!(result.is_err()); let err = result.unwrap_err().to_string(); diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 2ca597bd77..ce2208db71 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -10,11 +10,11 @@ use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::url::Url; -use deno_runtime::deno_fetch::reqwest; use crate::deno_dir::DenoDir; use crate::file_fetcher::CacheSetting; use crate::fs_util; +use crate::http_util::HttpClient; use crate::progress_bar::ProgressBar; use super::registry::NpmPackageVersionDistInfo; @@ -315,6 +315,7 @@ impl ReadonlyNpmCache { pub struct NpmCache { readonly: ReadonlyNpmCache, cache_setting: CacheSetting, + http_client: HttpClient, progress_bar: ProgressBar, } @@ -322,11 +323,13 @@ impl NpmCache { pub fn from_deno_dir( dir: &DenoDir, cache_setting: CacheSetting, + http_client: HttpClient, progress_bar: ProgressBar, ) -> Self { Self { readonly: ReadonlyNpmCache::from_deno_dir(dir), cache_setting, + http_client, progress_bar, } } @@ -383,7 +386,7 @@ impl NpmCache { } let _guard = self.progress_bar.update(&dist.tarball); - let response = reqwest::get(&dist.tarball).await?; + let response = self.http_client.get(&dist.tarball).send().await?; if response.status() == 404 { bail!("Could not find npm package tarball at: {}", dist.tarball); diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 066c136ee5..3f0a841655 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -18,12 +18,12 @@ use deno_core::serde::Deserialize; use deno_core::serde_json; use deno_core::url::Url; use deno_runtime::colors; -use deno_runtime::deno_fetch::reqwest; use serde::Serialize; use crate::file_fetcher::CacheSetting; use crate::fs_util; use crate::http_cache::CACHE_PERM; +use crate::http_util::HttpClient; use crate::progress_bar::ProgressBar; use super::cache::NpmCache; @@ -249,6 +249,7 @@ impl RealNpmRegistryApi { base_url: Url, cache: NpmCache, cache_setting: CacheSetting, + http_client: HttpClient, progress_bar: ProgressBar, ) -> Self { Self(Arc::new(RealNpmRegistryApiInner { @@ -256,6 +257,7 @@ impl RealNpmRegistryApi { cache, mem_cache: Default::default(), cache_setting, + http_client, progress_bar, })) } @@ -285,6 +287,7 @@ struct RealNpmRegistryApiInner { cache: NpmCache, mem_cache: Mutex>>>, cache_setting: CacheSetting, + http_client: HttpClient, progress_bar: ProgressBar, } @@ -420,7 +423,7 @@ impl RealNpmRegistryApiInner { let package_url = self.get_package_url(name); let _guard = self.progress_bar.update(package_url.as_str()); - let response = match reqwest::get(package_url).await { + let response = match self.http_client.get(package_url).send().await { Ok(response) => response, Err(err) => { // attempt to use the local cache diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 9e66bdb662..a193adc53d 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -19,6 +19,7 @@ use crate::graph_util::graph_lock_or_exit; use crate::graph_util::GraphData; use crate::graph_util::ModuleEntry; use crate::http_cache; +use crate::http_util::HttpClient; use crate::lockfile::as_maybe_locker; use crate::lockfile::Lockfile; use crate::node; @@ -157,15 +158,18 @@ impl ProcState { let root_cert_store = cli_options.resolve_root_cert_store()?; let cache_usage = cli_options.cache_setting(); let progress_bar = ProgressBar::default(); + let http_client = HttpClient::new( + Some(root_cert_store.clone()), + cli_options + .unsafely_ignore_certificate_errors() + .map(ToOwned::to_owned), + )?; let file_fetcher = FileFetcher::new( http_cache, cache_usage, !cli_options.no_remote(), - Some(root_cert_store.clone()), + http_client.clone(), blob_store.clone(), - cli_options - .unsafely_ignore_certificate_errors() - .map(ToOwned::to_owned), Some(progress_bar.clone()), )?; @@ -216,12 +220,14 @@ impl ProcState { let npm_cache = NpmCache::from_deno_dir( &dir, cli_options.cache_setting(), + http_client.clone(), progress_bar.clone(), ); let api = RealNpmRegistryApi::new( registry_url, npm_cache.clone(), cli_options.cache_setting(), + http_client, progress_bar.clone(), ); let maybe_lockfile = lockfile.as_ref().cloned();