Add gzip, brotli and ETag support for file fetcher (#3597)

Cargo.lock

@@ -1,5 +1,10 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+[[package]]
+name = "adler32"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "aho-corasick"
 version = "0.7.6"
@@ -36,6 +41,18 @@ name = "arrayvec"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "async-compression"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.13"
@@ -98,6 +115,24 @@ dependencies = [
  "constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "brotli-sys"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "brotli2"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "bumpalo"
 version = "2.6.0"
@@ -186,6 +221,14 @@ name = "core-foundation-sys"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "crc32fast"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.6.6"
@@ -210,6 +253,7 @@ dependencies = [
  "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -355,6 +399,17 @@ dependencies = [
  "synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "flate2"
+version = "1.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
+ "miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.6"
@@ -683,6 +738,14 @@ dependencies = [
  "unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "mio"
 version = "0.6.21"
@@ -1009,6 +1072,7 @@ name = "reqwest"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
+ "async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1738,12 +1802,14 @@ dependencies = [
 ]
 
 [metadata]
+"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
 "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
 "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
 "checksum anyhow 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
 "checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff"
 "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee"
 "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
+"checksum async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c5c52622726d68ec35fec88edfb4ccb862d4f3b3bfa4af2f45142e69ef9b220"
 "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
@@ -1752,6 +1818,8 @@ dependencies = [
 "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 "checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
+"checksum brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
+"checksum brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
 "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
 "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
 "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
@@ -1765,6 +1833,7 @@ dependencies = [
 "checksum constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120"
 "checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d"
 "checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
+"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
 "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
 "checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113"
 "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3"
@@ -1776,6 +1845,7 @@ dependencies = [
 "checksum encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28"
 "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
 "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
+"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
 "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
 "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
 "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
@@ -1813,6 +1883,7 @@ dependencies = [
 "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
 "checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
 "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
+"checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625"
 "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
 "checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3"
 "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"

cli/Cargo.toml

@@ -30,6 +30,7 @@ atty = "0.2.13"
 base64 = "0.11.0"
 bytes = "0.5.3"
 byteorder = "1.3.2"
+brotli2 = "0.3.2"
 clap = "2.33.0"
 dirs = "2.0.2"
 dlopen = "0.1.8"
@@ -44,7 +45,7 @@ log = "0.4.8"
 rand = "0.7.2"
 regex = "1.3.1"
 remove_dir_all = "0.5.2"
-reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream"] }
+reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream", "gzip"] }
 ring = "0.16.9"
 rustyline = "5.0.6"
 serde = { version = "1.0.104", features = ["derive"] }

cli/file_fetcher.rs

@@ -393,10 +393,21 @@ impl SourceFileFetcher {
     let download_job = self.progress.add("Download", &module_url.to_string());
     let dir = self.clone();
     let module_url = module_url.clone();
+    let headers = self.get_source_code_headers(&module_url);
+    let module_etag = headers.etag;
 
     // Single pass fetch, either yields code or yields redirect.
     let f = async move {
-      match http_util::fetch_string_once(&module_url).await? {
+      match http_util::fetch_string_once(&module_url, module_etag).await? {
+        FetchOnceResult::NotModified => {
+          let source_file =
+            dir.fetch_cached_remote_source(&module_url)?.unwrap();
+
+          // Explicit drop to keep reference alive until future completes.
+          drop(download_job);
+
+          Ok(source_file)
+        }
         FetchOnceResult::Redirect(new_module_url) => {
           // If redirects, update module_name and filename for next looped call.
           dir
@@ -404,6 +415,7 @@ impl SourceFileFetcher {
               &module_url,
               None,
               Some(new_module_url.to_string()),
+              None,
             )
             .unwrap();
 
@@ -420,13 +432,14 @@ impl SourceFileFetcher {
             )
             .await
         }
-        FetchOnceResult::Code(source, maybe_content_type) => {
+        FetchOnceResult::Code(source, maybe_content_type, etag) => {
           // We land on the code.
           dir
             .save_source_code_headers(
               &module_url,
               maybe_content_type.clone(),
               None,
+              etag,
             )
             .unwrap();
 
@@ -501,6 +514,7 @@ impl SourceFileFetcher {
     url: &Url,
     mime_type: Option<String>,
     redirect_to: Option<String>,
+    etag: Option<String>,
   ) -> std::io::Result<()> {
     let cache_key = self
       .deps_cache
@@ -513,6 +527,7 @@ impl SourceFileFetcher {
     let headers = SourceCodeHeaders {
       mime_type,
       redirect_to,
+      etag,
     };
 
     let cache_filename = self.deps_cache.get_cache_filename(url);
@@ -634,10 +649,13 @@ pub struct SourceCodeHeaders {
   /// Where should we actually look for source code.
   /// This should be an absolute path!
   pub redirect_to: Option<String>,
+  /// ETag of the remote source file
+  pub etag: Option<String>,
 }
 
 static MIME_TYPE: &str = "mime_type";
 static REDIRECT_TO: &str = "redirect_to";
+static ETAG: &str = "etag";
 
 impl SourceCodeHeaders {
   pub fn from_json_string(headers_string: String) -> Self {
@@ -648,10 +666,12 @@ impl SourceCodeHeaders {
     if let Ok(headers_json) = maybe_headers_json {
       let mime_type = headers_json[MIME_TYPE].as_str().map(String::from);
       let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from);
+      let etag = headers_json[ETAG].as_str().map(String::from);
 
       return SourceCodeHeaders {
         mime_type,
         redirect_to,
+        etag,
       };
     }
 
@@ -688,6 +708,10 @@ impl SourceCodeHeaders {
       value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to));
     }
 
+    if let Some(etag) = &self.etag {
+      value_map.insert(ETAG.to_string(), json!(etag));
+    }
+
     if value_map.is_empty() {
       return Ok(None);
     }
@@ -808,21 +832,27 @@ mod tests {
     let _ = deno_fs::write_file(
       headers_filepath.as_path(),
       "{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}",
-      0o666
+      0o666,
     );
     let headers = fetcher.get_source_code_headers(&url);
 
     assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript");
     assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js");
+    assert_eq!(headers.etag, None);
 
     let _ = fetcher.save_source_code_headers(
       &url,
       Some("text/typescript".to_owned()),
       Some("http://deno.land/a.js".to_owned()),
+      Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()),
     );
     let headers2 = fetcher.get_source_code_headers(&url);
     assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript");
     assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js");
+    assert_eq!(
+      headers2.etag.unwrap(),
+      "W/\"04572f4749af993f4961a7e5daa1e4d5\""
+    );
   }
 
   #[test]
@@ -901,6 +931,7 @@ mod tests {
           &module_url_1,
           Some("application/json".to_owned()),
           None,
+          None,
         );
         fetcher_2.get_source_file_async(&module_url_1, true, false, false)
       })
@@ -976,6 +1007,7 @@ mod tests {
           &module_url,
           Some("text/typescript".to_owned()),
           None,
+          None,
         );
         fetcher.get_source_file_async(&module_url, true, false, false)
       })
@@ -1344,6 +1376,7 @@ mod tests {
           &module_url,
           Some("text/javascript".to_owned()),
           None,
+          None,
         );
         let result2 = fetcher.fetch_cached_remote_source(&module_url);
         assert!(result2.is_ok());
@@ -1386,6 +1419,7 @@ mod tests {
           &module_url,
           Some("text/javascript".to_owned()),
           None,
+          None,
         );
         let result2 = fetcher.fetch_cached_remote_source(&module_url);
         assert!(result2.is_ok());
@@ -1683,28 +1717,28 @@ mod tests {
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
-        Some("application/typescript")
+        Some("application/typescript"),
       ),
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
-        Some("application/javascript")
+        Some("application/javascript"),
       ),
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
-        Some("application/x-typescript")
+        Some("application/x-typescript"),
       ),
       msg::MediaType::TSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.tsx"),
-        Some("video/vnd.dlna.mpeg-tts")
+        Some("video/vnd.dlna.mpeg-tts"),
       ),
       msg::MediaType::TSX
     );
@@ -1715,21 +1749,21 @@ mod tests {
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
-        Some("application/javascript")
+        Some("application/javascript"),
       ),
       msg::MediaType::JSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
-        Some("application/x-typescript")
+        Some("application/x-typescript"),
       ),
       msg::MediaType::JSX
     );
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
-        Some("application/ecmascript")
+        Some("application/ecmascript"),
       ),
       msg::MediaType::JSX
     );
@@ -1740,7 +1774,7 @@ mod tests {
     assert_eq!(
       map_content_type(
         Path::new("foo/bar.jsx"),
-        Some("application/x-javascript")
+        Some("application/x-javascript"),
       ),
       msg::MediaType::JSX
     );
@@ -1758,4 +1792,53 @@ mod tests {
       .to_owned();
     assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n".as_bytes());
   }
+
+  #[test]
+  fn test_fetch_with_etag() {
+    let http_server_guard = crate::test_util::http_server();
+    let (_temp_dir, fetcher) = test_setup();
+    let module_url =
+      Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
+
+    let fut = async move {
+      let source = fetcher
+        .fetch_remote_source_async(&module_url, false, false, 1)
+        .await;
+      assert!(source.is_ok());
+      let source = source.unwrap();
+      assert_eq!(source.source_code, b"console.log('etag')");
+      assert_eq!(&(source.media_type), &msg::MediaType::JavaScript);
+
+      let headers = fetcher.get_source_code_headers(&module_url);
+      assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string()));
+
+      let header_path = fetcher.deps_cache.location.join(
+        fetcher
+          .deps_cache
+          .get_cache_filename_with_extension(&module_url, "headers.json"),
+      );
+
+      let modified1 = header_path.metadata().unwrap().modified().unwrap();
+
+      // Forcibly change the contents of the cache file and request
+      // it again with the cache parameters turned off.
+      // If the fetched content changes, the cached content is used.
+      fetcher
+        .save_source_code(&module_url, "changed content")
+        .unwrap();
+      let cached_source = fetcher
+        .fetch_remote_source_async(&module_url, false, false, 1)
+        .await
+        .unwrap();
+      assert_eq!(cached_source.source_code, b"changed content");
+
+      let modified2 = header_path.metadata().unwrap().modified().unwrap();
+
+      // Assert that the file has not been modified
+      assert_eq!(modified1, modified2);
+    };
+
+    tokio_util::run(fut);
+    drop(http_server_guard);
+  }
 }

cli/http_util.rs

@@ -2,20 +2,27 @@
 use crate::deno_error;
 use crate::deno_error::DenoError;
 use crate::version;
+use brotli2::read::BrotliDecoder;
 use bytes::Bytes;
 use deno_core::ErrBox;
 use futures::future::FutureExt;
 use reqwest;
-use reqwest::header::HeaderMap;
+use reqwest::header::ACCEPT_ENCODING;
+use reqwest::header::CONTENT_ENCODING;
 use reqwest::header::CONTENT_TYPE;
+use reqwest::header::ETAG;
+use reqwest::header::IF_NONE_MATCH;
 use reqwest::header::LOCATION;
 use reqwest::header::USER_AGENT;
+use reqwest::header::{HeaderMap, HeaderValue};
 use reqwest::redirect::Policy;
 use reqwest::Client;
 use reqwest::Response;
+use reqwest::StatusCode;
 use std::cmp::min;
 use std::future::Future;
 use std::io;
+use std::io::Read;
 use std::pin::Pin;
 use std::task::Context;
 use std::task::Poll;
@@ -29,10 +36,15 @@ lazy_static! {
       USER_AGENT,
       format!("Deno/{}", version::DENO).parse().unwrap(),
     );
+    // todo support brotli for fetch ops
+    headers.insert(
+     ACCEPT_ENCODING, HeaderValue::from_static("gzip")
+    );
     Client::builder()
       .redirect(Policy::none())
       .default_headers(headers)
       .use_rustls_tls()
+      .gzip(true)
       .build()
       .unwrap()
   };
@@ -41,7 +53,7 @@ lazy_static! {
 /// Get instance of async reqwest::Client. This client supports
 /// proxies and doesn't follow redirects.
 pub fn get_client() -> &'static Client {
-  &HTTP_CLIENT
+  &HTTP_CLIENT as &Client
 }
 
 /// Construct the next uri based on base uri and location header fragment
@@ -73,8 +85,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
 
 #[derive(Debug, PartialEq)]
 pub enum FetchOnceResult {
-  // (code, maybe_content_type)
+  // (code, maybe_content_type, etag)
-  Code(String, Option<String>),
+  Code(String, Option<String>, Option<String>),
+  NotModified,
   Redirect(Url),
 }
 
@@ -85,12 +98,25 @@ pub enum FetchOnceResult {
 /// yields Redirect(url).
 pub fn fetch_string_once(
   url: &Url,
+  cached_etag: Option<String>,
 ) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> {
   let url = url.clone();
-  let client = get_client();
+  let client: &Client = get_client();
 
   let fut = async move {
-    let response = client.get(url.clone()).send().await?;
+    let mut request = client
+      .get(url.clone())
+      .header(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
+
+    if let Some(etag) = cached_etag {
+      let if_none_match_val = HeaderValue::from_str(&etag).unwrap();
+      request = request.header(IF_NONE_MATCH, if_none_match_val);
+    }
+    let response = request.send().await?;
+
+    if response.status() == StatusCode::NOT_MODIFIED {
+      return Ok(FetchOnceResult::NotModified);
+    }
 
     if response.status().is_redirection() {
       let location_string = response
@@ -120,8 +146,33 @@ pub fn fetch_string_once(
       .get(CONTENT_TYPE)
       .map(|content_type| content_type.to_str().unwrap().to_owned());
 
-    let body = response.text().await?;
+    let etag = response
-    return Ok(FetchOnceResult::Code(body, content_type));
+      .headers()
+      .get(ETAG)
+      .map(|etag| etag.to_str().unwrap().to_owned());
+
+    let content_encoding = response
+      .headers()
+      .get(CONTENT_ENCODING)
+      .map(|content_encoding| content_encoding.to_str().unwrap().to_owned());
+
+    let body;
+    if let Some(content_encoding) = content_encoding {
+      body = match content_encoding {
+        _ if content_encoding == "br" => {
+          let full_bytes = response.bytes().await?;
+          let mut decoder = BrotliDecoder::new(full_bytes.as_ref());
+          let mut body = String::new();
+          decoder.read_to_string(&mut body)?;
+          body
+        }
+        _ => response.text().await?,
+      }
+    } else {
+      body = response.text().await?;
+    }
+
+    return Ok(FetchOnceResult::Code(body, content_type, etag));
   };
 
   fut.boxed()
@@ -215,10 +266,93 @@ mod tests {
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
 
-    let fut = fetch_string_once(&url).map(|result| match result {
+    let fut = fetch_string_once(&url, None).map(|result| match result {
-      Ok(FetchOnceResult::Code(code, maybe_content_type)) => {
+      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
         assert!(!code.is_empty());
         assert_eq!(maybe_content_type, Some("application/json".to_string()));
+        assert_eq!(etag, None)
+      }
+      _ => panic!(),
+    });
+
+    tokio_util::run(fut);
+    drop(http_server_guard);
+  }
+
+  #[test]
+  fn test_fetch_gzip() {
+    let http_server_guard = crate::test_util::http_server();
+    // Relies on external http server. See tools/http_server.py
+    let url = Url::parse(
+      "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
+    )
+    .unwrap();
+
+    let fut = fetch_string_once(&url, None).map(|result| match result {
+      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+        assert!(!code.is_empty());
+        assert_eq!(code, "console.log('gzip')");
+        assert_eq!(
+          maybe_content_type,
+          Some("application/javascript".to_string())
+        );
+        assert_eq!(etag, None);
+      }
+      _ => panic!(),
+    });
+
+    tokio_util::run(fut);
+    drop(http_server_guard);
+  }
+
+  #[test]
+  fn test_fetch_with_etag() {
+    let http_server_guard = crate::test_util::http_server();
+    let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
+
+    let fut = async move {
+      fetch_string_once(&url, None)
+        .map(|result| match result {
+          Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+            assert!(!code.is_empty());
+            assert_eq!(code, "console.log('etag')");
+            assert_eq!(
+              maybe_content_type,
+              Some("application/javascript".to_string())
+            );
+            assert_eq!(etag, Some("33a64df551425fcc55e".to_string()));
+          }
+          _ => panic!(),
+        })
+        .await;
+
+      let res =
+        fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await;
+      assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
+    };
+
+    tokio_util::run(fut);
+    drop(http_server_guard);
+  }
+
+  #[test]
+  fn test_fetch_brotli() {
+    let http_server_guard = crate::test_util::http_server();
+    // Relies on external http server. See tools/http_server.py
+    let url = Url::parse(
+      "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
+    )
+    .unwrap();
+
+    let fut = fetch_string_once(&url, None).map(|result| match result {
+      Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+        assert!(!code.is_empty());
+        assert_eq!(code, "console.log('brotli');");
+        assert_eq!(
+          maybe_content_type,
+          Some("application/javascript".to_string())
+        );
+        assert_eq!(etag, None);
       }
       _ => panic!(),
     });
@@ -236,7 +370,7 @@ mod tests {
     // Dns resolver substitutes `127.0.0.1` with `localhost`
     let target_url =
       Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
-    let fut = fetch_string_once(&url).map(move |result| match result {
+    let fut = fetch_string_once(&url, None).map(move |result| match result {
       Ok(FetchOnceResult::Redirect(url)) => {
         assert_eq!(url, target_url);
       }

cli/tests/053_import_compression.out

@@ -0,0 +1,3 @@
+gzip
+brotli
+console.log('gzip')

cli/tests/053_import_compression/brotli

@@ -0,0 +1,2 @@
+‹
+€console.log('brotli');

cli/tests/053_import_compression/brotli.header

@@ -0,0 +1,3 @@
+Content-Encoding: br
+Content-Type: application/javascript
+Content-Length: 26

cli/tests/053_import_compression/gziped.header

@@ -0,0 +1,3 @@
+Content-Encoding: gzip
+Content-Type: application/javascript
+Content-Length: 39

cli/tests/053_import_compression/main.ts

@@ -0,0 +1,8 @@
+import "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped";
+import "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli";
+
+console.log(
+  await fetch(
+    "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped"
+  ).then(res => res.text())
+);

cli/tests/integration_tests.rs

@@ -661,6 +661,12 @@ itest!(top_level_for_await_ts {
   output: "top_level_for_await.out",
 });
 
+itest!(_053_import_compression {
+  args: "run --reload --allow-net 053_import_compression/main.ts",
+  output: "053_import_compression.out",
+  http_server: true,
+});
+
 mod util {
   use deno::colors::strip_ansi_codes;
   pub use deno::test_util::*;

tools/http_server.py

@@ -31,6 +31,42 @@ class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
 
 class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
     def do_GET(self):
+
+        # Check if there is a custom header configuration ending
+        # with ".header" before sending the file
+        maybe_header_file_path = "./" + self.path + ".header"
+        if os.path.exists(maybe_header_file_path):
+            self.protocol_version = 'HTTP/1.1'
+            self.send_response(200, 'OK')
+
+            f = open(maybe_header_file_path)
+            for line in f:
+                kv = line.split(": ")
+                self.send_header(kv[0].strip(), kv[1].strip())
+            f.close()
+            self.end_headers()
+
+            body = open("./" + self.path)
+            self.wfile.write(body.read())
+            body.close()
+            return
+
+        if "etag_script.ts" in self.path:
+            self.protocol_version = 'HTTP/1.1'
+            if_not_match = self.headers.getheader('if-none-match')
+            if if_not_match == "33a64df551425fcc55e":
+                self.send_response(304, 'Not Modified')
+                self.send_header('Content-type', 'application/javascript')
+                self.send_header('ETag', '33a64df551425fcc55e')
+                self.end_headers()
+            else:
+                self.send_response(200, 'OK')
+                self.send_header('Content-type', 'application/javascript')
+                self.send_header('ETag', '33a64df551425fcc55e')
+                self.end_headers()
+                self.wfile.write(bytes("console.log('etag')"))
+            return
+
         if "multipart_form_data.txt" in self.path:
             self.protocol_version = 'HTTP/1.1'
             self.send_response(200, 'OK')