mirror of
https://github.com/denoland/deno.git
synced 2024-12-25 00:29:09 -05:00
Add gzip, brotli and ETag support for file fetcher (#3597)
This commit is contained in:
EnokMan
Bartek Iwańczuk
parent
0d0ad360d3
commit
1ea06f9c84
12 changed files with 373 additions and 23 deletions
71
Cargo.lock
generated
71
Cargo.lock
generated
|
|
@ -1,5 +1,10 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "adler32"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.6"
|
||||
|
|
@ -36,6 +41,18 @@ name = "arrayvec"
|
|||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.13"
|
||||
|
|
@ -98,6 +115,24 @@ dependencies = [
|
|||
"constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli-sys"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli2"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "2.6.0"
|
||||
|
|
@ -186,6 +221,14 @@ name = "core-foundation-sys"
|
|||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.6.6"
|
||||
|
|
@ -210,6 +253,7 @@ dependencies = [
|
|||
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -355,6 +399,17 @@ dependencies = [
|
|||
"synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.6"
|
||||
|
|
@ -683,6 +738,14 @@ dependencies = [
|
|||
"unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.3.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.6.21"
|
||||
|
|
@ -1009,6 +1072,7 @@ name = "reqwest"
|
|||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -1738,12 +1802,14 @@ dependencies = [
|
|||
]
|
||||
|
||||
[metadata]
|
||||
"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
|
||||
"checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d"
|
||||
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
"checksum anyhow 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
|
||||
"checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff"
|
||||
"checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee"
|
||||
"checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
|
||||
"checksum async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c5c52622726d68ec35fec88edfb4ccb862d4f3b3bfa4af2f45142e69ef9b220"
|
||||
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
|
||||
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
|
||||
"checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
|
||||
|
|
@ -1752,6 +1818,8 @@ dependencies = [
|
|||
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
"checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a"
|
||||
"checksum brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
|
||||
"checksum brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
|
||||
"checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"
|
||||
"checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5"
|
||||
"checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
|
||||
|
|
@ -1765,6 +1833,7 @@ dependencies = [
|
|||
"checksum constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120"
|
||||
"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d"
|
||||
"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
|
||||
"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
|
||||
"checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6"
|
||||
"checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113"
|
||||
"checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3"
|
||||
|
|
@ -1776,6 +1845,7 @@ dependencies = [
|
|||
"checksum encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28"
|
||||
"checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9"
|
||||
"checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08"
|
||||
"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
|
||||
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
|
||||
"checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||
"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
||||
|
|
@ -1813,6 +1883,7 @@ dependencies = [
|
|||
"checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e"
|
||||
"checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf"
|
||||
"checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599"
|
||||
"checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625"
|
||||
"checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f"
|
||||
"checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3"
|
||||
"checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ atty = "0.2.13"
|
|||
base64 = "0.11.0"
|
||||
bytes = "0.5.3"
|
||||
byteorder = "1.3.2"
|
||||
brotli2 = "0.3.2"
|
||||
clap = "2.33.0"
|
||||
dirs = "2.0.2"
|
||||
dlopen = "0.1.8"
|
||||
|
|
@ -44,7 +45,7 @@ log = "0.4.8"
|
|||
rand = "0.7.2"
|
||||
regex = "1.3.1"
|
||||
remove_dir_all = "0.5.2"
|
||||
reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream"] }
|
||||
reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream", "gzip"] }
|
||||
ring = "0.16.9"
|
||||
rustyline = "5.0.6"
|
||||
serde = { version = "1.0.104", features = ["derive"] }
|
||||
|
|
|
|||
|
|
@ -393,10 +393,21 @@ impl SourceFileFetcher {
|
|||
let download_job = self.progress.add("Download", &module_url.to_string());
|
||||
let dir = self.clone();
|
||||
let module_url = module_url.clone();
|
||||
let headers = self.get_source_code_headers(&module_url);
|
||||
let module_etag = headers.etag;
|
||||
|
||||
// Single pass fetch, either yields code or yields redirect.
|
||||
let f = async move {
|
||||
match http_util::fetch_string_once(&module_url).await? {
|
||||
match http_util::fetch_string_once(&module_url, module_etag).await? {
|
||||
FetchOnceResult::NotModified => {
|
||||
let source_file =
|
||||
dir.fetch_cached_remote_source(&module_url)?.unwrap();
|
||||
|
||||
// Explicit drop to keep reference alive until future completes.
|
||||
drop(download_job);
|
||||
|
||||
Ok(source_file)
|
||||
}
|
||||
FetchOnceResult::Redirect(new_module_url) => {
|
||||
// If redirects, update module_name and filename for next looped call.
|
||||
dir
|
||||
|
|
@ -404,6 +415,7 @@ impl SourceFileFetcher {
|
|||
&module_url,
|
||||
None,
|
||||
Some(new_module_url.to_string()),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -420,13 +432,14 @@ impl SourceFileFetcher {
|
|||
)
|
||||
.await
|
||||
}
|
||||
FetchOnceResult::Code(source, maybe_content_type) => {
|
||||
FetchOnceResult::Code(source, maybe_content_type, etag) => {
|
||||
// We land on the code.
|
||||
dir
|
||||
.save_source_code_headers(
|
||||
&module_url,
|
||||
maybe_content_type.clone(),
|
||||
None,
|
||||
etag,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
|
|
@ -501,6 +514,7 @@ impl SourceFileFetcher {
|
|||
url: &Url,
|
||||
mime_type: Option<String>,
|
||||
redirect_to: Option<String>,
|
||||
etag: Option<String>,
|
||||
) -> std::io::Result<()> {
|
||||
let cache_key = self
|
||||
.deps_cache
|
||||
|
|
@ -513,6 +527,7 @@ impl SourceFileFetcher {
|
|||
let headers = SourceCodeHeaders {
|
||||
mime_type,
|
||||
redirect_to,
|
||||
etag,
|
||||
};
|
||||
|
||||
let cache_filename = self.deps_cache.get_cache_filename(url);
|
||||
|
|
@ -634,10 +649,13 @@ pub struct SourceCodeHeaders {
|
|||
/// Where should we actually look for source code.
|
||||
/// This should be an absolute path!
|
||||
pub redirect_to: Option<String>,
|
||||
/// ETag of the remote source file
|
||||
pub etag: Option<String>,
|
||||
}
|
||||
|
||||
static MIME_TYPE: &str = "mime_type";
|
||||
static REDIRECT_TO: &str = "redirect_to";
|
||||
static ETAG: &str = "etag";
|
||||
|
||||
impl SourceCodeHeaders {
|
||||
pub fn from_json_string(headers_string: String) -> Self {
|
||||
|
|
@ -648,10 +666,12 @@ impl SourceCodeHeaders {
|
|||
if let Ok(headers_json) = maybe_headers_json {
|
||||
let mime_type = headers_json[MIME_TYPE].as_str().map(String::from);
|
||||
let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from);
|
||||
let etag = headers_json[ETAG].as_str().map(String::from);
|
||||
|
||||
return SourceCodeHeaders {
|
||||
mime_type,
|
||||
redirect_to,
|
||||
etag,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -688,6 +708,10 @@ impl SourceCodeHeaders {
|
|||
value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to));
|
||||
}
|
||||
|
||||
if let Some(etag) = &self.etag {
|
||||
value_map.insert(ETAG.to_string(), json!(etag));
|
||||
}
|
||||
|
||||
if value_map.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
|
@ -808,21 +832,27 @@ mod tests {
|
|||
let _ = deno_fs::write_file(
|
||||
headers_filepath.as_path(),
|
||||
"{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}",
|
||||
0o666
|
||||
0o666,
|
||||
);
|
||||
let headers = fetcher.get_source_code_headers(&url);
|
||||
|
||||
assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript");
|
||||
assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js");
|
||||
assert_eq!(headers.etag, None);
|
||||
|
||||
let _ = fetcher.save_source_code_headers(
|
||||
&url,
|
||||
Some("text/typescript".to_owned()),
|
||||
Some("http://deno.land/a.js".to_owned()),
|
||||
Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()),
|
||||
);
|
||||
let headers2 = fetcher.get_source_code_headers(&url);
|
||||
assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript");
|
||||
assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js");
|
||||
assert_eq!(
|
||||
headers2.etag.unwrap(),
|
||||
"W/\"04572f4749af993f4961a7e5daa1e4d5\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -901,6 +931,7 @@ mod tests {
|
|||
&module_url_1,
|
||||
Some("application/json".to_owned()),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
fetcher_2.get_source_file_async(&module_url_1, true, false, false)
|
||||
})
|
||||
|
|
@ -976,6 +1007,7 @@ mod tests {
|
|||
&module_url,
|
||||
Some("text/typescript".to_owned()),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
fetcher.get_source_file_async(&module_url, true, false, false)
|
||||
})
|
||||
|
|
@ -1344,6 +1376,7 @@ mod tests {
|
|||
&module_url,
|
||||
Some("text/javascript".to_owned()),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let result2 = fetcher.fetch_cached_remote_source(&module_url);
|
||||
assert!(result2.is_ok());
|
||||
|
|
@ -1386,6 +1419,7 @@ mod tests {
|
|||
&module_url,
|
||||
Some("text/javascript".to_owned()),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
let result2 = fetcher.fetch_cached_remote_source(&module_url);
|
||||
assert!(result2.is_ok());
|
||||
|
|
@ -1683,28 +1717,28 @@ mod tests {
|
|||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.tsx"),
|
||||
Some("application/typescript")
|
||||
Some("application/typescript"),
|
||||
),
|
||||
msg::MediaType::TSX
|
||||
);
|
||||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.tsx"),
|
||||
Some("application/javascript")
|
||||
Some("application/javascript"),
|
||||
),
|
||||
msg::MediaType::TSX
|
||||
);
|
||||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.tsx"),
|
||||
Some("application/x-typescript")
|
||||
Some("application/x-typescript"),
|
||||
),
|
||||
msg::MediaType::TSX
|
||||
);
|
||||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.tsx"),
|
||||
Some("video/vnd.dlna.mpeg-tts")
|
||||
Some("video/vnd.dlna.mpeg-tts"),
|
||||
),
|
||||
msg::MediaType::TSX
|
||||
);
|
||||
|
|
@ -1715,21 +1749,21 @@ mod tests {
|
|||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.jsx"),
|
||||
Some("application/javascript")
|
||||
Some("application/javascript"),
|
||||
),
|
||||
msg::MediaType::JSX
|
||||
);
|
||||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.jsx"),
|
||||
Some("application/x-typescript")
|
||||
Some("application/x-typescript"),
|
||||
),
|
||||
msg::MediaType::JSX
|
||||
);
|
||||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.jsx"),
|
||||
Some("application/ecmascript")
|
||||
Some("application/ecmascript"),
|
||||
),
|
||||
msg::MediaType::JSX
|
||||
);
|
||||
|
|
@ -1740,7 +1774,7 @@ mod tests {
|
|||
assert_eq!(
|
||||
map_content_type(
|
||||
Path::new("foo/bar.jsx"),
|
||||
Some("application/x-javascript")
|
||||
Some("application/x-javascript"),
|
||||
),
|
||||
msg::MediaType::JSX
|
||||
);
|
||||
|
|
@ -1758,4 +1792,53 @@ mod tests {
|
|||
.to_owned();
|
||||
assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n".as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_with_etag() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
let (_temp_dir, fetcher) = test_setup();
|
||||
let module_url =
|
||||
Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
|
||||
|
||||
let fut = async move {
|
||||
let source = fetcher
|
||||
.fetch_remote_source_async(&module_url, false, false, 1)
|
||||
.await;
|
||||
assert!(source.is_ok());
|
||||
let source = source.unwrap();
|
||||
assert_eq!(source.source_code, b"console.log('etag')");
|
||||
assert_eq!(&(source.media_type), &msg::MediaType::JavaScript);
|
||||
|
||||
let headers = fetcher.get_source_code_headers(&module_url);
|
||||
assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string()));
|
||||
|
||||
let header_path = fetcher.deps_cache.location.join(
|
||||
fetcher
|
||||
.deps_cache
|
||||
.get_cache_filename_with_extension(&module_url, "headers.json"),
|
||||
);
|
||||
|
||||
let modified1 = header_path.metadata().unwrap().modified().unwrap();
|
||||
|
||||
// Forcibly change the contents of the cache file and request
|
||||
// it again with the cache parameters turned off.
|
||||
// If the fetched content changes, the cached content is used.
|
||||
fetcher
|
||||
.save_source_code(&module_url, "changed content")
|
||||
.unwrap();
|
||||
let cached_source = fetcher
|
||||
.fetch_remote_source_async(&module_url, false, false, 1)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(cached_source.source_code, b"changed content");
|
||||
|
||||
let modified2 = header_path.metadata().unwrap().modified().unwrap();
|
||||
|
||||
// Assert that the file has not been modified
|
||||
assert_eq!(modified1, modified2);
|
||||
};
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
156
cli/http_util.rs
156
cli/http_util.rs
|
|
@ -2,20 +2,27 @@
|
|||
use crate::deno_error;
|
||||
use crate::deno_error::DenoError;
|
||||
use crate::version;
|
||||
use brotli2::read::BrotliDecoder;
|
||||
use bytes::Bytes;
|
||||
use deno_core::ErrBox;
|
||||
use futures::future::FutureExt;
|
||||
use reqwest;
|
||||
use reqwest::header::HeaderMap;
|
||||
use reqwest::header::ACCEPT_ENCODING;
|
||||
use reqwest::header::CONTENT_ENCODING;
|
||||
use reqwest::header::CONTENT_TYPE;
|
||||
use reqwest::header::ETAG;
|
||||
use reqwest::header::IF_NONE_MATCH;
|
||||
use reqwest::header::LOCATION;
|
||||
use reqwest::header::USER_AGENT;
|
||||
use reqwest::header::{HeaderMap, HeaderValue};
|
||||
use reqwest::redirect::Policy;
|
||||
use reqwest::Client;
|
||||
use reqwest::Response;
|
||||
use reqwest::StatusCode;
|
||||
use std::cmp::min;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::io::Read;
|
||||
use std::pin::Pin;
|
||||
use std::task::Context;
|
||||
use std::task::Poll;
|
||||
|
|
@ -29,10 +36,15 @@ lazy_static! {
|
|||
USER_AGENT,
|
||||
format!("Deno/{}", version::DENO).parse().unwrap(),
|
||||
);
|
||||
// todo support brotli for fetch ops
|
||||
headers.insert(
|
||||
ACCEPT_ENCODING, HeaderValue::from_static("gzip")
|
||||
);
|
||||
Client::builder()
|
||||
.redirect(Policy::none())
|
||||
.default_headers(headers)
|
||||
.use_rustls_tls()
|
||||
.gzip(true)
|
||||
.build()
|
||||
.unwrap()
|
||||
};
|
||||
|
|
@ -41,7 +53,7 @@ lazy_static! {
|
|||
/// Get instance of async reqwest::Client. This client supports
|
||||
/// proxies and doesn't follow redirects.
|
||||
pub fn get_client() -> &'static Client {
|
||||
&HTTP_CLIENT
|
||||
&HTTP_CLIENT as &Client
|
||||
}
|
||||
|
||||
/// Construct the next uri based on base uri and location header fragment
|
||||
|
|
@ -73,8 +85,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
|
|||
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub enum FetchOnceResult {
|
||||
// (code, maybe_content_type)
|
||||
Code(String, Option<String>),
|
||||
// (code, maybe_content_type, etag)
|
||||
Code(String, Option<String>, Option<String>),
|
||||
NotModified,
|
||||
Redirect(Url),
|
||||
}
|
||||
|
||||
|
|
@ -85,12 +98,25 @@ pub enum FetchOnceResult {
|
|||
/// yields Redirect(url).
|
||||
pub fn fetch_string_once(
|
||||
url: &Url,
|
||||
cached_etag: Option<String>,
|
||||
) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> {
|
||||
let url = url.clone();
|
||||
let client = get_client();
|
||||
let client: &Client = get_client();
|
||||
|
||||
let fut = async move {
|
||||
let response = client.get(url.clone()).send().await?;
|
||||
let mut request = client
|
||||
.get(url.clone())
|
||||
.header(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
|
||||
|
||||
if let Some(etag) = cached_etag {
|
||||
let if_none_match_val = HeaderValue::from_str(&etag).unwrap();
|
||||
request = request.header(IF_NONE_MATCH, if_none_match_val);
|
||||
}
|
||||
let response = request.send().await?;
|
||||
|
||||
if response.status() == StatusCode::NOT_MODIFIED {
|
||||
return Ok(FetchOnceResult::NotModified);
|
||||
}
|
||||
|
||||
if response.status().is_redirection() {
|
||||
let location_string = response
|
||||
|
|
@ -120,8 +146,33 @@ pub fn fetch_string_once(
|
|||
.get(CONTENT_TYPE)
|
||||
.map(|content_type| content_type.to_str().unwrap().to_owned());
|
||||
|
||||
let body = response.text().await?;
|
||||
return Ok(FetchOnceResult::Code(body, content_type));
|
||||
let etag = response
|
||||
.headers()
|
||||
.get(ETAG)
|
||||
.map(|etag| etag.to_str().unwrap().to_owned());
|
||||
|
||||
let content_encoding = response
|
||||
.headers()
|
||||
.get(CONTENT_ENCODING)
|
||||
.map(|content_encoding| content_encoding.to_str().unwrap().to_owned());
|
||||
|
||||
let body;
|
||||
if let Some(content_encoding) = content_encoding {
|
||||
body = match content_encoding {
|
||||
_ if content_encoding == "br" => {
|
||||
let full_bytes = response.bytes().await?;
|
||||
let mut decoder = BrotliDecoder::new(full_bytes.as_ref());
|
||||
let mut body = String::new();
|
||||
decoder.read_to_string(&mut body)?;
|
||||
body
|
||||
}
|
||||
_ => response.text().await?,
|
||||
}
|
||||
} else {
|
||||
body = response.text().await?;
|
||||
}
|
||||
|
||||
return Ok(FetchOnceResult::Code(body, content_type, etag));
|
||||
};
|
||||
|
||||
fut.boxed()
|
||||
|
|
@ -215,10 +266,93 @@ mod tests {
|
|||
let url =
|
||||
Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type)) => {
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(maybe_content_type, Some("application/json".to_string()));
|
||||
assert_eq!(etag, None)
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_gzip() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
let url = Url::parse(
|
||||
"http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('gzip')");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_with_etag() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
|
||||
|
||||
let fut = async move {
|
||||
fetch_string_once(&url, None)
|
||||
.map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('etag')");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, Some("33a64df551425fcc55e".to_string()));
|
||||
}
|
||||
_ => panic!(),
|
||||
})
|
||||
.await;
|
||||
|
||||
let res =
|
||||
fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await;
|
||||
assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
|
||||
};
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fetch_brotli() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
// Relies on external http server. See tools/http_server.py
|
||||
let url = Url::parse(
|
||||
"http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let fut = fetch_string_once(&url, None).map(|result| match result {
|
||||
Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
|
||||
assert!(!code.is_empty());
|
||||
assert_eq!(code, "console.log('brotli');");
|
||||
assert_eq!(
|
||||
maybe_content_type,
|
||||
Some("application/javascript".to_string())
|
||||
);
|
||||
assert_eq!(etag, None);
|
||||
}
|
||||
_ => panic!(),
|
||||
});
|
||||
|
|
@ -236,7 +370,7 @@ mod tests {
|
|||
// Dns resolver substitutes `127.0.0.1` with `localhost`
|
||||
let target_url =
|
||||
Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
|
||||
let fut = fetch_string_once(&url).map(move |result| match result {
|
||||
let fut = fetch_string_once(&url, None).map(move |result| match result {
|
||||
Ok(FetchOnceResult::Redirect(url)) => {
|
||||
assert_eq!(url, target_url);
|
||||
}
|
||||
|
|
|
|||
3
cli/tests/053_import_compression.out
Normal file
3
cli/tests/053_import_compression.out
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
gzip
|
||||
brotli
|
||||
console.log('gzip')
|
||||
2
cli/tests/053_import_compression/brotli
Normal file
2
cli/tests/053_import_compression/brotli
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
‹
|
||||
€console.log('brotli');
|
||||
3
cli/tests/053_import_compression/brotli.header
Normal file
3
cli/tests/053_import_compression/brotli.header
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Content-Encoding: br
|
||||
Content-Type: application/javascript
|
||||
Content-Length: 26
|
||||
BIN
cli/tests/053_import_compression/gziped
Normal file
BIN
cli/tests/053_import_compression/gziped
Normal file
Binary file not shown.
3
cli/tests/053_import_compression/gziped.header
Normal file
3
cli/tests/053_import_compression/gziped.header
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
Content-Encoding: gzip
|
||||
Content-Type: application/javascript
|
||||
Content-Length: 39
|
||||
8
cli/tests/053_import_compression/main.ts
Normal file
8
cli/tests/053_import_compression/main.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
import "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped";
|
||||
import "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli";
|
||||
|
||||
console.log(
|
||||
await fetch(
|
||||
"http://127.0.0.1:4545/cli/tests/053_import_compression/gziped"
|
||||
).then(res => res.text())
|
||||
);
|
||||
|
|
@ -661,6 +661,12 @@ itest!(top_level_for_await_ts {
|
|||
output: "top_level_for_await.out",
|
||||
});
|
||||
|
||||
itest!(_053_import_compression {
|
||||
args: "run --reload --allow-net 053_import_compression/main.ts",
|
||||
output: "053_import_compression.out",
|
||||
http_server: true,
|
||||
});
|
||||
|
||||
mod util {
|
||||
use deno::colors::strip_ansi_codes;
|
||||
pub use deno::test_util::*;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,42 @@ class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
|
|||
|
||||
class ContentTypeHandler(QuietSimpleHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
|
||||
# Check if there is a custom header configuration ending
|
||||
# with ".header" before sending the file
|
||||
maybe_header_file_path = "./" + self.path + ".header"
|
||||
if os.path.exists(maybe_header_file_path):
|
||||
self.protocol_version = 'HTTP/1.1'
|
||||
self.send_response(200, 'OK')
|
||||
|
||||
f = open(maybe_header_file_path)
|
||||
for line in f:
|
||||
kv = line.split(": ")
|
||||
self.send_header(kv[0].strip(), kv[1].strip())
|
||||
f.close()
|
||||
self.end_headers()
|
||||
|
||||
body = open("./" + self.path)
|
||||
self.wfile.write(body.read())
|
||||
body.close()
|
||||
return
|
||||
|
||||
if "etag_script.ts" in self.path:
|
||||
self.protocol_version = 'HTTP/1.1'
|
||||
if_not_match = self.headers.getheader('if-none-match')
|
||||
if if_not_match == "33a64df551425fcc55e":
|
||||
self.send_response(304, 'Not Modified')
|
||||
self.send_header('Content-type', 'application/javascript')
|
||||
self.send_header('ETag', '33a64df551425fcc55e')
|
||||
self.end_headers()
|
||||
else:
|
||||
self.send_response(200, 'OK')
|
||||
self.send_header('Content-type', 'application/javascript')
|
||||
self.send_header('ETag', '33a64df551425fcc55e')
|
||||
self.end_headers()
|
||||
self.wfile.write(bytes("console.log('etag')"))
|
||||
return
|
||||
|
||||
if "multipart_form_data.txt" in self.path:
|
||||
self.protocol_version = 'HTTP/1.1'
|
||||
self.send_response(200, 'OK')
|
||||
|
|
|
|||
Loading…
Reference in a new issue