mirror of
https://github.com/denoland/deno.git
synced 2024-12-24 08:09:08 -05:00
perf: share http client in file fetcher (#3683)
This commit is contained in:
parent
91757f63fd
commit
1de02b0643
3 changed files with 66 additions and 51 deletions
|
@ -5,6 +5,7 @@ use crate::deno_error::ErrorKind;
|
|||
use crate::deno_error::GetErrorKind;
|
||||
use crate::disk_cache::DiskCache;
|
||||
use crate::http_util;
|
||||
use crate::http_util::create_http_client;
|
||||
use crate::http_util::FetchOnceResult;
|
||||
use crate::msg;
|
||||
use crate::progress::Progress;
|
||||
|
@ -12,6 +13,7 @@ use deno_core::ErrBox;
|
|||
use deno_core::ModuleSpecifier;
|
||||
use futures::future::Either;
|
||||
use futures::future::FutureExt;
|
||||
use reqwest;
|
||||
use serde_json;
|
||||
use std;
|
||||
use std::collections::HashMap;
|
||||
|
@ -75,6 +77,7 @@ pub struct SourceFileFetcher {
|
|||
use_disk_cache: bool,
|
||||
no_remote: bool,
|
||||
cached_only: bool,
|
||||
http_client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl SourceFileFetcher {
|
||||
|
@ -94,6 +97,7 @@ impl SourceFileFetcher {
|
|||
use_disk_cache,
|
||||
no_remote,
|
||||
cached_only,
|
||||
http_client: create_http_client(),
|
||||
};
|
||||
|
||||
Ok(file_fetcher)
|
||||
|
@ -395,10 +399,12 @@ impl SourceFileFetcher {
|
|||
let module_url = module_url.clone();
|
||||
let headers = self.get_source_code_headers(&module_url);
|
||||
let module_etag = headers.etag;
|
||||
|
||||
let http_client = self.http_client.clone();
|
||||
// Single pass fetch, either yields code or yields redirect.
|
||||
let f = async move {
|
||||
match http_util::fetch_string_once(&module_url, module_etag).await? {
|
||||
match http_util::fetch_string_once(http_client, &module_url, module_etag)
|
||||
.await?
|
||||
{
|
||||
FetchOnceResult::NotModified => {
|
||||
let source_file =
|
||||
dir.fetch_cached_remote_source(&module_url)?.unwrap();
|
||||
|
|
103
cli/http_util.rs
103
cli/http_util.rs
|
@ -31,7 +31,7 @@ use url::Url;
|
|||
|
||||
/// Create new instance of async reqwest::Client. This client supports
|
||||
/// proxies and doesn't follow redirects.
|
||||
pub fn get_client() -> Client {
|
||||
pub fn create_http_client() -> Client {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(
|
||||
USER_AGENT,
|
||||
|
@ -86,11 +86,11 @@ pub enum FetchOnceResult {
|
|||
/// If redirect occurs, does not follow and
|
||||
/// yields Redirect(url).
|
||||
pub fn fetch_string_once(
|
||||
client: Client,
|
||||
url: &Url,
|
||||
cached_etag: Option<String>,
|
||||
) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> {
|
||||
let url = url.clone();
|
||||
let client = get_client();
|
||||
|
||||
let fut = async move {
|
||||
let mut request = client
|
||||
|
@ -254,15 +254,16 @@ mod tests {
|
|||
// Relies on external http server. See tools/http_server.py
|
||||
let url =
|
||||
Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(maybe_content_type, Some("application/json".to_string()));
|
||||
assert_eq!(etag, None)
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
let client = create_http_client();
|
||||
let fut =
|
||||
fetch_string_once(client, &url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(maybe_content_type, Some("application/json".to_string()));
|
||||
assert_eq!(etag, None)
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
|
@ -276,19 +277,20 @@ mod tests {
|
|||
"http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('gzip')");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
let client = create_http_client();
|
||||
let fut =
|
||||
fetch_string_once(client, &url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('gzip')");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
|
@ -298,9 +300,9 @@ mod tests {
|
|||
fn test_fetch_with_etag() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
|
||||
|
||||
let client = create_http_client();
|
||||
let fut = async move {
|
||||
fetch_string_once(&url, None)
|
||||
fetch_string_once(client.clone(), &url, None)
|
||||
.map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
|
@ -315,8 +317,12 @@ mod tests {
|
|||
})
|
||||
.await;
|
||||
|
||||
let res =
|
||||
fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await;
|
||||
let res = fetch_string_once(
|
||||
client,
|
||||
&url,
|
||||
Some("33a64df551425fcc55e".to_string()),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
|
||||
};
|
||||
|
||||
|
@ -332,19 +338,20 @@ mod tests {
|
|||
"http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('brotli');");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
let client = create_http_client();
|
||||
let fut =
|
||||
fetch_string_once(client, &url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('brotli');");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
|
@ -359,12 +366,14 @@ mod tests {
|
|||
// Dns resolver substitutes `127.0.0.1` with `localhost`
|
||||
let target_url =
|
||||
Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
|
||||
let fut = fetch_string_once(&url, None).map(move |result| match result {
|
||||
Ok(FetchOnceResult::Redirect(url)) => {
|
||||
assert_eq!(url, target_url);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
let client = create_http_client();
|
||||
let fut =
|
||||
fetch_string_once(client, &url, None).map(move |result| match result {
|
||||
Ok(FetchOnceResult::Redirect(url)) => {
|
||||
assert_eq!(url, target_url);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
|
||||
use super::dispatch_json::{Deserialize, JsonOp, Value};
|
||||
use super::io::StreamResource;
|
||||
use crate::http_util::{get_client, HttpBody};
|
||||
use crate::http_util::{create_http_client, HttpBody};
|
||||
use crate::ops::json_op;
|
||||
use crate::state::ThreadSafeState;
|
||||
use deno_core::*;
|
||||
|
@ -31,7 +31,7 @@ pub fn op_fetch(
|
|||
let args: FetchArgs = serde_json::from_value(args)?;
|
||||
let url = args.url;
|
||||
|
||||
let client = get_client();
|
||||
let client = create_http_client();
|
||||
|
||||
let method = match args.method {
|
||||
Some(method_str) => Method::from_bytes(method_str.as_bytes())?,
|
||||
|
|
Loading…
Reference in a new issue