perf: share http client in file fetcher (#3683)

cli/file_fetcher.rs

@@ -5,6 +5,7 @@ use crate::deno_error::ErrorKind;
 use crate::deno_error::GetErrorKind;
 use crate::disk_cache::DiskCache;
 use crate::http_util;
+use crate::http_util::create_http_client;
 use crate::http_util::FetchOnceResult;
 use crate::msg;
 use crate::progress::Progress;
@@ -12,6 +13,7 @@ use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
 use futures::future::Either;
 use futures::future::FutureExt;
+use reqwest;
 use serde_json;
 use std;
 use std::collections::HashMap;
@@ -75,6 +77,7 @@ pub struct SourceFileFetcher {
   use_disk_cache: bool,
   no_remote: bool,
   cached_only: bool,
+  http_client: reqwest::Client,
 }
 
 impl SourceFileFetcher {
@@ -94,6 +97,7 @@ impl SourceFileFetcher {
       use_disk_cache,
       no_remote,
       cached_only,
+      http_client: create_http_client(),
     };
 
     Ok(file_fetcher)
@@ -395,10 +399,12 @@ impl SourceFileFetcher {
     let module_url = module_url.clone();
     let headers = self.get_source_code_headers(&module_url);
     let module_etag = headers.etag;
-
+    let http_client = self.http_client.clone();
     // Single pass fetch, either yields code or yields redirect.
     let f = async move {
-      match http_util::fetch_string_once(&module_url, module_etag).await? {
+      match http_util::fetch_string_once(http_client, &module_url, module_etag)
+        .await?
+      {
         FetchOnceResult::NotModified => {
           let source_file =
             dir.fetch_cached_remote_source(&module_url)?.unwrap();

cli/http_util.rs

@@ -31,7 +31,7 @@ use url::Url;
 
 /// Create new instance of async reqwest::Client. This client supports
 /// proxies and doesn't follow redirects.
-pub fn get_client() -> Client {
+pub fn create_http_client() -> Client {
   let mut headers = HeaderMap::new();
   headers.insert(
     USER_AGENT,
@@ -86,11 +86,11 @@ pub enum FetchOnceResult {
 /// If redirect occurs, does not follow and
 /// yields Redirect(url).
 pub fn fetch_string_once(
+  client: Client,
   url: &Url,
   cached_etag: Option<String>,
 ) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> {
   let url = url.clone();
-  let client = get_client();
 
   let fut = async move {
     let mut request = client
@@ -254,15 +254,16 @@ mod tests {
     // Relies on external http server. See tools/http_server.py
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
-
+    let client = create_http_client();
-    let fut = fetch_string_once(&url, None).map(|result| match result {
+    let fut =
-      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+      fetch_string_once(client, &url, None).map(|result| match result {
-        assert!(!code.is_empty());
+        Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
-        assert_eq!(maybe_content_type, Some("application/json".to_string()));
+          assert!(!code.is_empty());
-        assert_eq!(etag, None)
+          assert_eq!(maybe_content_type, Some("application/json".to_string()));
-      }
+          assert_eq!(etag, None)
-      _ => panic!(),
+        }
-    });
+        _ => panic!(),
+      });
 
     tokio_util::run(fut);
     drop(http_server_guard);
@@ -276,19 +277,20 @@ mod tests {
       "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
     )
     .unwrap();
-
+    let client = create_http_client();
-    let fut = fetch_string_once(&url, None).map(|result| match result {
+    let fut =
-      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+      fetch_string_once(client, &url, None).map(|result| match result {
-        assert!(!code.is_empty());
+        Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
-        assert_eq!(code, "console.log('gzip')");
+          assert!(!code.is_empty());
-        assert_eq!(
+          assert_eq!(code, "console.log('gzip')");
-          maybe_content_type,
+          assert_eq!(
-          Some("application/javascript".to_string())
+            maybe_content_type,
-        );
+            Some("application/javascript".to_string())
-        assert_eq!(etag, None);
+          );
-      }
+          assert_eq!(etag, None);
-      _ => panic!(),
+        }
-    });
+        _ => panic!(),
+      });
 
     tokio_util::run(fut);
     drop(http_server_guard);
@@ -298,9 +300,9 @@ mod tests {
   fn test_fetch_with_etag() {
     let http_server_guard = crate::test_util::http_server();
     let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
-
+    let client = create_http_client();
     let fut = async move {
-      fetch_string_once(&url, None)
+      fetch_string_once(client.clone(), &url, None)
         .map(|result| match result {
           Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
             assert!(!code.is_empty());
@@ -315,8 +317,12 @@ mod tests {
         })
         .await;
 
-      let res =
+      let res = fetch_string_once(
-        fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await;
+        client,
+        &url,
+        Some("33a64df551425fcc55e".to_string()),
+      )
+      .await;
       assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
     };
 
@@ -332,19 +338,20 @@ mod tests {
       "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
     )
     .unwrap();
-
+    let client = create_http_client();
-    let fut = fetch_string_once(&url, None).map(|result| match result {
+    let fut =
-      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+      fetch_string_once(client, &url, None).map(|result| match result {
-        assert!(!code.is_empty());
+        Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
-        assert_eq!(code, "console.log('brotli');");
+          assert!(!code.is_empty());
-        assert_eq!(
+          assert_eq!(code, "console.log('brotli');");
-          maybe_content_type,
+          assert_eq!(
-          Some("application/javascript".to_string())
+            maybe_content_type,
-        );
+            Some("application/javascript".to_string())
-        assert_eq!(etag, None);
+          );
-      }
+          assert_eq!(etag, None);
-      _ => panic!(),
+        }
-    });
+        _ => panic!(),
+      });
 
     tokio_util::run(fut);
     drop(http_server_guard);
@@ -359,12 +366,14 @@ mod tests {
     // Dns resolver substitutes `127.0.0.1` with `localhost`
     let target_url =
       Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
-    let fut = fetch_string_once(&url, None).map(move |result| match result {
+    let client = create_http_client();
-      Ok(FetchOnceResult::Redirect(url)) => {
+    let fut =
-        assert_eq!(url, target_url);
+      fetch_string_once(client, &url, None).map(move |result| match result {
-      }
+        Ok(FetchOnceResult::Redirect(url)) => {
-      _ => panic!(),
+          assert_eq!(url, target_url);
-    });
+        }
+        _ => panic!(),
+      });
 
     tokio_util::run(fut);
     drop(http_server_guard);

cli/ops/fetch.rs

@@ -1,7 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 use super::dispatch_json::{Deserialize, JsonOp, Value};
 use super::io::StreamResource;
-use crate::http_util::{get_client, HttpBody};
+use crate::http_util::{create_http_client, HttpBody};
 use crate::ops::json_op;
 use crate::state::ThreadSafeState;
 use deno_core::*;
@@ -31,7 +31,7 @@ pub fn op_fetch(
   let args: FetchArgs = serde_json::from_value(args)?;
   let url = args.url;
 
-  let client = get_client();
+  let client = create_http_client();
 
   let method = match args.method {
     Some(method_str) => Method::from_bytes(method_str.as_bytes())?,