From 05f838a57cc5b5e7b262e1640baa4d93dfb0df28 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Aug 2023 10:23:02 -0400 Subject: [PATCH] refactor: use deno_cache_dir crate (#20092) Uses https://github.com/denoland/deno_cache/pull/26 --- Cargo.lock | 175 ++-- cli/Cargo.toml | 1 + cli/cache/disk_cache.rs | 2 +- cli/cache/http_cache/common.rs | 42 - cli/cache/http_cache/global.rs | 296 ------- cli/cache/http_cache/local.rs | 1425 -------------------------------- cli/cache/http_cache/mod.rs | 77 -- cli/cache/mod.rs | 55 +- cli/factory.rs | 1 + cli/file_fetcher.rs | 37 +- cli/lsp/completions.rs | 10 +- cli/lsp/diagnostics.rs | 14 +- cli/lsp/documents.rs | 6 +- cli/lsp/language_server.rs | 10 +- cli/lsp/registries.rs | 5 +- cli/lsp/tsc.rs | 11 +- 16 files changed, 221 insertions(+), 1946 deletions(-) delete mode 100644 cli/cache/http_cache/common.rs delete mode 100644 cli/cache/http_cache/global.rs delete mode 100644 cli/cache/http_cache/local.rs delete mode 100644 cli/cache/http_cache/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 399fe53cc4..81a07018ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,9 +174,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "arrayvec" @@ -207,7 +207,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "synstructure", ] @@ -219,7 +219,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -231,7 +231,7 @@ checksum = "c704e2f6ee1a98223f5a7629a6ef0f3decb3b552ed282889dc957edff98ce1e6" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -268,7 +268,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -279,8 +279,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -302,7 +302,7 @@ checksum = "7862e21c893d65a1650125d157eaeec691439379a1cee17ee49031b79236ada4" dependencies = [ "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -725,7 +725,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -807,6 +807,7 @@ dependencies = [ "data-url", "deno_ast", "deno_bench_util", + "deno_cache_dir", "deno_core", "deno_doc", "deno_emit", @@ -889,7 +890,7 @@ checksum = "3c65c2ffdafc1564565200967edc4851c7b55422d3913466688907efd05ea26f" dependencies = [ "deno-proc-macro-rules-macros", "proc-macro2 1.0.66", - "syn 2.0.22", + "syn 2.0.28", ] [[package]] @@ -900,8 +901,8 @@ checksum = "3047b312b7451e3190865713a4dd6e1f821aed614ada219766ebc3024a690435" dependencies = [ "once_cell", "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -968,6 +969,25 @@ dependencies = [ "tokio", ] +[[package]] +name = "deno_cache_dir" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd2de39c0d15e801f5968334998336c159ac7e335f8acddd781982777195152" +dependencies = [ + "anyhow", + "deno_media_type", + "indexmap 2.0.0", + "log", + "once_cell", + "parking_lot 0.12.1", + "ring", + "serde", + "serde_json", + "thiserror", + "url", +] + [[package]] name = "deno_console" version = "0.114.0" @@ -1359,12 +1379,12 @@ dependencies = [ "pmutil 0.6.1", "proc-macro-crate", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "regex", "strum", "strum_macros", "syn 1.0.109", - "syn 2.0.22", + "syn 2.0.28", "thiserror", ] @@ -1575,7 +1595,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -1623,7 +1643,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -1759,7 +1779,7 @@ dependencies = [ "lazy_static", "proc-macro-error", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -1880,7 +1900,7 @@ checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -2068,7 +2088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cde5eb168cf5a056dd98f311cbfab7494c216394e4fb9eba0336827a8db93" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -2206,8 +2226,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -2644,6 +2664,7 @@ checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", "hashbrown 0.14.0", + "serde", ] [[package]] @@ -2723,7 +2744,7 @@ dependencies = [ "Inflector", "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -2829,7 +2850,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8edfc11b8f56ce85e207e62ea21557cfa09bb24a8f6b04ae181b086ff8611c22" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "regex", "syn 1.0.109", ] @@ -3162,7 +3183,7 @@ name = "napi_sym" version = "0.44.0" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "serde", "serde_json", "syn 1.0.109", @@ -3331,9 +3352,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "opaque-debug" @@ -3594,7 +3615,7 @@ dependencies = [ "phf_shared", "proc-macro-hack", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -3623,7 +3644,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -3684,7 +3705,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -3695,8 +3716,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -3762,7 +3783,7 @@ checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "version_check", ] @@ -3774,7 +3795,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "version_check", ] @@ -3862,9 +3883,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2 1.0.66", ] @@ -4214,7 +4235,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "107c3d5d7f370ac09efa62a78375f94d94b8a33c61d8c278b96683fb4dbf2d8d" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -4378,9 +4399,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.157" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707de5fcf5df2b5788fca98dd7eab490bc2fd9b7ef1404defc462833b83f25ca" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] @@ -4406,22 +4427,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.157" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78997f4555c22a7971214540c4a661291970619afd56de19f77e0de86296e1e5" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] name = "serde_json" -version = "1.0.94" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ - "indexmap 1.9.2", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -4434,7 +4455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -4677,7 +4698,7 @@ dependencies = [ "phf_generator", "phf_shared", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", ] [[package]] @@ -4688,7 +4709,7 @@ checksum = "0090512bdfee4b56d82480d66c0fd8a6f53f0fe0f97e075e949b252acdd482e0" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -4716,9 +4737,9 @@ checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" dependencies = [ "heck", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "rustversion", - "syn 2.0.22", + "syn 2.0.28", ] [[package]] @@ -4820,7 +4841,7 @@ checksum = "7dadb9998d4f5fc36ef558ed5a092579441579ee8c6fcce84a5228cca9df4004" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -4869,7 +4890,7 @@ checksum = "bf4ee0caee1018808d94ecd09490cb7affd3d504b19aa11c49238f5fc4b54901" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -4965,7 +4986,7 @@ checksum = "984d5ac69b681fc5438f9abf82b0fda34fe04e119bc75f8213b7e01128c7c9a2" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -5096,7 +5117,7 @@ checksum = "0c20468634668c2bbab581947bb8c75c97158d5a6959f4ba33df20983b20b4f6" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -5133,7 +5154,7 @@ checksum = "3e582c3e3c2269238524923781df5be49e011dbe29cf7683a2215d600a562ea6" dependencies = [ "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -5156,7 +5177,7 @@ dependencies = [ "Inflector", "pmutil 0.5.3", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "swc_macros_common", "syn 1.0.109", ] @@ -5179,18 +5200,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.22" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "unicode-ident", ] @@ -5201,7 +5222,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -5309,22 +5330,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -5406,8 +5427,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", - "syn 2.0.22", + "quote 1.0.32", + "syn 2.0.28", ] [[package]] @@ -5545,7 +5566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebd99eec668d0a450c177acbc4d05e0d0d13b1f8d3db13cd706c52cbec4ac04" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -5574,7 +5595,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", ] @@ -5927,7 +5948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", ] [[package]] @@ -5989,7 +6010,7 @@ dependencies = [ "log", "once_cell", "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "wasm-bindgen-shared", ] @@ -6012,7 +6033,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote 1.0.28", + "quote 1.0.32", "wasm-bindgen-macro-support", ] @@ -6023,7 +6044,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -6377,7 +6398,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ "proc-macro2 1.0.66", - "quote 1.0.28", + "quote 1.0.32", "syn 1.0.109", "synstructure", ] diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 7b7066db7f..4896f240c2 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -46,6 +46,7 @@ winres.workspace = true [dependencies] deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] } +deno_cache_dir = "=0.5.0" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = "=0.64.0" deno_emit = "=0.25.0" diff --git a/cli/cache/disk_cache.rs b/cli/cache/disk_cache.rs index 6950c056a9..1375df6dca 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/cache/disk_cache.rs @@ -1,9 +1,9 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use super::http_cache::url_to_filename; use super::CACHE_PERM; use crate::util::fs::atomic_write_file; +use deno_cache_dir::url_to_filename; use deno_core::url::Host; use deno_core::url::Url; use std::ffi::OsStr; diff --git a/cli/cache/http_cache/common.rs b/cli/cache/http_cache/common.rs deleted file mode 100644 index 690412293e..0000000000 --- a/cli/cache/http_cache/common.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use std::path::Path; - -use deno_core::url::Url; - -pub fn base_url_to_filename_parts( - url: &Url, - port_separator: &str, -) -> Option> { - let mut out = Vec::with_capacity(2); - - let scheme = url.scheme(); - out.push(scheme.to_string()); - - match scheme { - "http" | "https" => { - let host = url.host_str().unwrap(); - let host_port = match url.port() { - // underscores are not allowed in domains, so adding one here is fine - Some(port) => format!("{host}{port_separator}{port}"), - None => host.to_string(), - }; - out.push(host_port); - } - "data" | "blob" => (), - scheme => { - log::debug!("Don't know how to create cache name for scheme: {}", scheme); - return None; - } - }; - - Some(out) -} - -pub fn read_file_bytes(path: &Path) -> std::io::Result>> { - match std::fs::read(path) { - Ok(s) => Ok(Some(s)), - Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err), - } -} diff --git a/cli/cache/http_cache/global.rs b/cli/cache/http_cache/global.rs deleted file mode 100644 index 5c77553a8f..0000000000 --- a/cli/cache/http_cache/global.rs +++ /dev/null @@ -1,296 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use std::io; -use std::path::Path; -use std::path::PathBuf; -use std::time::SystemTime; - -use deno_core::error::AnyError; -use deno_core::serde_json; -use deno_core::url::Url; -use thiserror::Error; - -use crate::cache::CACHE_PERM; -use crate::http_util::HeadersMap; -use crate::util; -use crate::util::fs::atomic_write_file; - -use super::common::base_url_to_filename_parts; -use super::common::read_file_bytes; -use super::CachedUrlMetadata; -use super::HttpCache; -use super::HttpCacheItemKey; - -#[derive(Debug, Error)] -#[error("Can't convert url (\"{}\") to filename.", .url)] -pub struct UrlToFilenameConversionError { - pub(super) url: String, -} - -/// Turn provided `url` into a hashed filename. -/// URLs can contain a lot of characters that cannot be used -/// in filenames (like "?", "#", ":"), so in order to cache -/// them properly they are deterministically hashed into ASCII -/// strings. -pub fn url_to_filename( - url: &Url, -) -> Result { - let Some(mut cache_filename) = base_url_to_filename(url) else { - return Err(UrlToFilenameConversionError { url: url.to_string() }); - }; - - let mut rest_str = url.path().to_string(); - if let Some(query) = url.query() { - rest_str.push('?'); - rest_str.push_str(query); - } - // NOTE: fragment is omitted on purpose - it's not taken into - // account when caching - it denotes parts of webpage, which - // in case of static resources doesn't make much sense - let hashed_filename = util::checksum::gen(&[rest_str.as_bytes()]); - cache_filename.push(hashed_filename); - Ok(cache_filename) -} - -// Turn base of url (scheme, hostname, port) into a valid filename. -/// This method replaces port part with a special string token (because -/// ":" cannot be used in filename on some platforms). -/// Ex: $DENO_DIR/deps/https/deno.land/ -fn base_url_to_filename(url: &Url) -> Option { - base_url_to_filename_parts(url, "_PORT").map(|parts| { - let mut out = PathBuf::new(); - for part in parts { - out.push(part); - } - out - }) -} - -#[derive(Debug)] -pub struct GlobalHttpCache(PathBuf); - -impl GlobalHttpCache { - pub fn new(path: PathBuf) -> Self { - assert!(path.is_absolute()); - Self(path) - } - - // Deprecated to discourage using this as where the file is stored and - // how it's stored should be an implementation detail of the cache. - #[deprecated(note = "Should only be used for deno info.")] - pub fn get_global_cache_location(&self) -> &PathBuf { - &self.0 - } - - // DEPRECATED: Where the file is stored and how it's stored should be an implementation - // detail of the cache. - #[deprecated(note = "Do not assume the cache will be stored at a file path.")] - pub fn get_global_cache_filepath( - &self, - url: &Url, - ) -> Result { - Ok(self.0.join(url_to_filename(url)?)) - } - - fn get_cache_filepath(&self, url: &Url) -> Result { - Ok(self.0.join(url_to_filename(url)?)) - } - - #[inline] - fn key_file_path<'a>(&self, key: &'a HttpCacheItemKey) -> &'a PathBuf { - // The key file path is always set for the global cache because - // the file will always exist, unlike the local cache, which won't - // have this for redirects. - key.file_path.as_ref().unwrap() - } -} - -impl HttpCache for GlobalHttpCache { - fn cache_item_key<'a>( - &self, - url: &'a Url, - ) -> Result, AnyError> { - Ok(HttpCacheItemKey { - #[cfg(debug_assertions)] - is_local_key: false, - url, - file_path: Some(self.get_cache_filepath(url)?), - }) - } - - fn contains(&self, url: &Url) -> bool { - let Ok(cache_filepath) = self.get_cache_filepath(url) else { - return false - }; - cache_filepath.is_file() - } - - fn read_modified_time( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(!key.is_local_key); - - match std::fs::metadata(self.key_file_path(key)) { - Ok(metadata) => Ok(Some(metadata.modified()?)), - Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(None), - Err(err) => Err(err.into()), - } - } - - fn set( - &self, - url: &Url, - headers: HeadersMap, - content: &[u8], - ) -> Result<(), AnyError> { - let cache_filepath = self.get_cache_filepath(url)?; - // Cache content - atomic_write_file(&cache_filepath, content, CACHE_PERM)?; - - let metadata = CachedUrlMetadata { - time: SystemTime::now(), - url: url.to_string(), - headers, - }; - write_metadata(&cache_filepath, &metadata)?; - - Ok(()) - } - - fn read_file_bytes( - &self, - key: &HttpCacheItemKey, - ) -> Result>, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(!key.is_local_key); - - Ok(read_file_bytes(self.key_file_path(key))?) - } - - fn read_metadata( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(!key.is_local_key); - - match read_metadata(self.key_file_path(key))? { - Some(metadata) => Ok(Some(metadata)), - None => Ok(None), - } - } -} - -fn read_metadata(path: &Path) -> Result, AnyError> { - let path = path.with_extension("metadata.json"); - match read_file_bytes(&path)? { - Some(metadata) => Ok(Some(serde_json::from_slice(&metadata)?)), - None => Ok(None), - } -} - -fn write_metadata( - path: &Path, - meta_data: &CachedUrlMetadata, -) -> Result<(), AnyError> { - let path = path.with_extension("metadata.json"); - let json = serde_json::to_string_pretty(meta_data)?; - atomic_write_file(&path, json, CACHE_PERM)?; - Ok(()) -} - -#[cfg(test)] -mod test { - use super::*; - use std::collections::HashMap; - use test_util::TempDir; - - #[test] - fn test_url_to_filename() { - let test_cases = [ - ("https://deno.land/x/foo.ts", "https/deno.land/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8"), - ( - "https://deno.land:8080/x/foo.ts", - "https/deno.land_PORT8080/2c0a064891b9e3fbe386f5d4a833bce5076543f5404613656042107213a7bbc8", - ), - ("https://deno.land/", "https/deno.land/8a5edab282632443219e051e4ade2d1d5bbc671c781051bf1437897cbdfea0f1"), - ( - "https://deno.land/?asdf=qwer", - "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", - ), - // should be the same as case above, fragment (#qwer) is ignored - // when hashing - ( - "https://deno.land/?asdf=qwer#qwer", - "https/deno.land/e4edd1f433165141015db6a823094e6bd8f24dd16fe33f2abd99d34a0a21a3c0", - ), - ( - "data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=", - "data/c21c7fc382b2b0553dc0864aa81a3acacfb7b3d1285ab5ae76da6abec213fb37", - ), - ( - "data:text/plain,Hello%2C%20Deno!", - "data/967374e3561d6741234131e342bf5c6848b70b13758adfe23ee1a813a8131818", - ) - ]; - - for (url, expected) in test_cases.iter() { - let u = Url::parse(url).unwrap(); - let p = url_to_filename(&u).unwrap(); - assert_eq!(p, PathBuf::from(expected)); - } - } - - #[test] - fn test_create_cache() { - let dir = TempDir::new(); - let cache_path = dir.path().join("foobar"); - // HttpCache should be created lazily on first use: - // when zipping up a local project with no external dependencies - // "$DENO_DIR/deps" is empty. When unzipping such project - // "$DENO_DIR/deps" might not get restored and in situation - // when directory is owned by root we might not be able - // to create that directory. However if it's not needed it - // doesn't make sense to return error in such specific scenarios. - // For more details check issue: - // https://github.com/denoland/deno/issues/5688 - let cache = GlobalHttpCache::new(cache_path.to_path_buf()); - assert!(!cache.0.exists()); - let url = Url::parse("http://example.com/foo/bar.js").unwrap(); - cache - .set(&url, HeadersMap::new(), b"hello world") - .expect("Failed to add to cache"); - assert!(cache_path.is_dir()); - assert!(cache.get_cache_filepath(&url).unwrap().is_file()); - } - - #[test] - fn test_get_set() { - let dir = TempDir::new(); - let cache = GlobalHttpCache::new(dir.path().to_path_buf()); - let url = Url::parse("https://deno.land/x/welcome.ts").unwrap(); - let mut headers = HashMap::new(); - headers.insert( - "content-type".to_string(), - "application/javascript".to_string(), - ); - headers.insert("etag".to_string(), "as5625rqdsfb".to_string()); - let content = b"Hello world"; - let r = cache.set(&url, headers, content); - eprintln!("result {r:?}"); - assert!(r.is_ok()); - let key = cache.cache_item_key(&url).unwrap(); - let content = - String::from_utf8(cache.read_file_bytes(&key).unwrap().unwrap()).unwrap(); - let headers = cache.read_metadata(&key).unwrap().unwrap().headers; - assert_eq!(content, "Hello world"); - assert_eq!( - headers.get("content-type").unwrap(), - "application/javascript" - ); - assert_eq!(headers.get("etag").unwrap(), "as5625rqdsfb"); - assert_eq!(headers.get("foobar"), None); - } -} diff --git a/cli/cache/http_cache/local.rs b/cli/cache/http_cache/local.rs deleted file mode 100644 index 04883b3ba8..0000000000 --- a/cli/cache/http_cache/local.rs +++ /dev/null @@ -1,1425 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use std::borrow::Cow; -use std::collections::HashMap; -use std::collections::HashSet; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::SystemTime; - -use deno_ast::MediaType; -use deno_core::error::AnyError; -use deno_core::parking_lot::RwLock; -use deno_core::url::Url; -use indexmap::IndexMap; -use once_cell::sync::Lazy; - -use crate::cache::CACHE_PERM; -use crate::http_util::HeadersMap; -use crate::util; -use crate::util::fs::atomic_write_file; - -use super::common::base_url_to_filename_parts; -use super::common::read_file_bytes; -use super::global::GlobalHttpCache; -use super::global::UrlToFilenameConversionError; -use super::CachedUrlMetadata; -use super::HttpCache; -use super::HttpCacheItemKey; - -/// A vendor/ folder http cache for the lsp that provides functionality -/// for doing a reverse mapping. -#[derive(Debug)] -pub struct LocalLspHttpCache { - cache: LocalHttpCache, -} - -impl LocalLspHttpCache { - pub fn new(path: PathBuf, global_cache: Arc) -> Self { - assert!(path.is_absolute()); - let manifest = LocalCacheManifest::new_for_lsp(path.join("manifest.json")); - Self { - cache: LocalHttpCache { - path, - manifest, - global_cache, - }, - } - } - - pub fn get_file_url(&self, url: &Url) -> Option { - let sub_path = { - let data = self.cache.manifest.data.read(); - let maybe_content_type = - data.get(url).and_then(|d| d.content_type_header()); - url_to_local_sub_path(url, maybe_content_type).ok()? - }; - let path = sub_path.as_path_from_root(&self.cache.path); - if path.exists() { - Url::from_file_path(path).ok() - } else { - None - } - } - - pub fn get_remote_url(&self, path: &Path) -> Option { - let Ok(path) = path.strip_prefix(&self.cache.path) else { - return None; // not in this directory - }; - let components = path - .components() - .map(|c| c.as_os_str().to_string_lossy()) - .collect::>(); - if components - .last() - .map(|c| c.starts_with('#')) - .unwrap_or(false) - { - // the file itself will have an entry in the manifest - let data = self.cache.manifest.data.read(); - data.get_reverse_mapping(path) - } else if let Some(last_index) = - components.iter().rposition(|c| c.starts_with('#')) - { - // get the mapping to the deepest hashed directory and - // then add the remaining path components to the url - let dir_path: PathBuf = components[..last_index + 1].iter().fold( - PathBuf::new(), - |mut path, c| { - path.push(c.as_ref()); - path - }, - ); - let dir_url = self - .cache - .manifest - .data - .read() - .get_reverse_mapping(&dir_path)?; - let file_url = - dir_url.join(&components[last_index + 1..].join("/")).ok()?; - Some(file_url) - } else { - // we can work backwards from the path to the url - let mut parts = Vec::new(); - for (i, part) in path.components().enumerate() { - let part = part.as_os_str().to_string_lossy(); - if i == 0 { - let mut result = String::new(); - let part = if let Some(part) = part.strip_prefix("http_") { - result.push_str("http://"); - part - } else { - result.push_str("https://"); - &part - }; - if let Some((domain, port)) = part.rsplit_once('_') { - result.push_str(&format!("{}:{}", domain, port)); - } else { - result.push_str(part); - } - parts.push(result); - } else { - parts.push(part.to_string()); - } - } - Url::parse(&parts.join("/")).ok() - } - } -} - -impl HttpCache for LocalLspHttpCache { - fn cache_item_key<'a>( - &self, - url: &'a Url, - ) -> Result, AnyError> { - self.cache.cache_item_key(url) - } - - fn contains(&self, url: &Url) -> bool { - self.cache.contains(url) - } - - fn set( - &self, - url: &Url, - headers: HeadersMap, - content: &[u8], - ) -> Result<(), AnyError> { - self.cache.set(url, headers, content) - } - - fn read_modified_time( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - self.cache.read_modified_time(key) - } - - fn read_file_bytes( - &self, - key: &HttpCacheItemKey, - ) -> Result>, AnyError> { - self.cache.read_file_bytes(key) - } - - fn read_metadata( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - self.cache.read_metadata(key) - } -} - -#[derive(Debug)] -pub struct LocalHttpCache { - path: PathBuf, - manifest: LocalCacheManifest, - global_cache: Arc, -} - -impl LocalHttpCache { - pub fn new(path: PathBuf, global_cache: Arc) -> Self { - assert!(path.is_absolute()); - let manifest = LocalCacheManifest::new(path.join("manifest.json")); - Self { - path, - manifest, - global_cache, - } - } - - /// Copies the file from the global cache to the local cache returning - /// if the data was successfully copied to the local cache. - fn check_copy_global_to_local(&self, url: &Url) -> Result { - let global_key = self.global_cache.cache_item_key(url)?; - let Some(metadata) = self.global_cache.read_metadata(&global_key)? else { - return Ok(false); - }; - - let local_path = - url_to_local_sub_path(url, headers_content_type(&metadata.headers))?; - - if !metadata.is_redirect() { - let Some(cached_bytes) = self.global_cache.read_file_bytes(&global_key)? else { - return Ok(false); - }; - - let local_file_path = local_path.as_path_from_root(&self.path); - // if we're here, then this will be set - atomic_write_file(&local_file_path, cached_bytes, CACHE_PERM)?; - } - - self - .manifest - .insert_data(local_path, url.clone(), metadata.headers); - - Ok(true) - } - - fn get_url_metadata_checking_global_cache( - &self, - url: &Url, - ) -> Result, AnyError> { - if let Some(metadata) = self.manifest.get_metadata(url) { - Ok(Some(metadata)) - } else if self.check_copy_global_to_local(url)? { - // try again now that it's saved - Ok(self.manifest.get_metadata(url)) - } else { - Ok(None) - } - } -} - -impl HttpCache for LocalHttpCache { - fn cache_item_key<'a>( - &self, - url: &'a Url, - ) -> Result, AnyError> { - Ok(HttpCacheItemKey { - #[cfg(debug_assertions)] - is_local_key: true, - url, - file_path: None, // need to compute this every time - }) - } - - fn contains(&self, url: &Url) -> bool { - self.manifest.get_metadata(url).is_some() - } - - fn read_modified_time( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(key.is_local_key); - - self - .get_url_metadata_checking_global_cache(key.url) - .map(|m| m.map(|m| m.time)) - } - - fn set( - &self, - url: &Url, - headers: crate::http_util::HeadersMap, - content: &[u8], - ) -> Result<(), AnyError> { - let is_redirect = headers.contains_key("location"); - let sub_path = url_to_local_sub_path(url, headers_content_type(&headers))?; - - if !is_redirect { - // Cache content - atomic_write_file( - &sub_path.as_path_from_root(&self.path), - content, - CACHE_PERM, - )?; - } - - self.manifest.insert_data(sub_path, url.clone(), headers); - - Ok(()) - } - - fn read_file_bytes( - &self, - key: &HttpCacheItemKey, - ) -> Result>, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(key.is_local_key); - - let metadata = self.get_url_metadata_checking_global_cache(key.url)?; - match metadata { - Some(data) => { - if data.is_redirect() { - // return back an empty file for redirect - Ok(Some(Vec::new())) - } else { - // if it's not a redirect, then it should have a file path - let cache_file_path = url_to_local_sub_path( - key.url, - headers_content_type(&data.headers), - )? - .as_path_from_root(&self.path); - Ok(read_file_bytes(&cache_file_path)?) - } - } - None => Ok(None), - } - } - - fn read_metadata( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError> { - #[cfg(debug_assertions)] - debug_assert!(key.is_local_key); - - self.get_url_metadata_checking_global_cache(key.url) - } -} - -pub(super) struct LocalCacheSubPath { - pub has_hash: bool, - pub parts: Vec, -} - -impl LocalCacheSubPath { - pub fn as_path_from_root(&self, root_path: &Path) -> PathBuf { - let mut path = root_path.to_path_buf(); - for part in &self.parts { - path.push(part); - } - path - } - - pub fn as_relative_path(&self) -> PathBuf { - let mut path = PathBuf::with_capacity(self.parts.len()); - for part in &self.parts { - path.push(part); - } - path - } -} - -fn headers_content_type(headers: &HeadersMap) -> Option<&str> { - headers.get("content-type").map(|s| s.as_str()) -} - -fn url_to_local_sub_path( - url: &Url, - content_type: Option<&str>, -) -> Result { - // https://stackoverflow.com/a/31976060/188246 - static FORBIDDEN_CHARS: Lazy> = Lazy::new(|| { - HashSet::from(['?', '<', '>', ':', '*', '|', '\\', ':', '"', '\'', '/']) - }); - // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file - static FORBIDDEN_WINDOWS_NAMES: Lazy> = - Lazy::new(|| { - let set = HashSet::from([ - "con", "prn", "aux", "nul", "com0", "com1", "com2", "com3", "com4", - "com5", "com6", "com7", "com8", "com9", "lpt0", "lpt1", "lpt2", "lpt3", - "lpt4", "lpt5", "lpt6", "lpt7", "lpt8", "lpt9", - ]); - // ensure everything is lowercase because we'll be comparing - // lowercase filenames against this - debug_assert!(set.iter().all(|s| s.to_lowercase() == *s)); - set - }); - - fn has_forbidden_chars(segment: &str) -> bool { - segment.chars().any(|c| { - let is_uppercase = c.is_ascii_alphabetic() && !c.is_ascii_lowercase(); - FORBIDDEN_CHARS.contains(&c) - // do not allow uppercase letters in order to make this work - // the same on case insensitive file systems - || is_uppercase - }) - } - - fn has_known_extension(path: &str) -> bool { - let path = path.to_lowercase(); - path.ends_with(".js") - || path.ends_with(".ts") - || path.ends_with(".jsx") - || path.ends_with(".tsx") - || path.ends_with(".mts") - || path.ends_with(".mjs") - || path.ends_with(".json") - || path.ends_with(".wasm") - } - - fn get_extension(url: &Url, content_type: Option<&str>) -> &'static str { - MediaType::from_specifier_and_content_type(url, content_type) - .as_ts_extension() - } - - fn short_hash(data: &str, last_ext: Option<&str>) -> String { - // This function is a bit of a balancing act between readability - // and avoiding collisions. - let hash = util::checksum::gen(&[data.as_bytes()]); - // keep the paths short because of windows path limit - const MAX_LENGTH: usize = 20; - let mut sub = String::with_capacity(MAX_LENGTH); - for c in data.chars().take(MAX_LENGTH) { - // don't include the query string (only use it in the hash) - if c == '?' { - break; - } - if FORBIDDEN_CHARS.contains(&c) { - sub.push('_'); - } else { - sub.extend(c.to_lowercase()); - } - } - let sub = match last_ext { - Some(ext) => sub.strip_suffix(ext).unwrap_or(&sub), - None => &sub, - }; - let ext = last_ext.unwrap_or(""); - if sub.is_empty() { - format!("#{}{}", &hash[..7], ext) - } else { - format!("#{}_{}{}", &sub, &hash[..5], ext) - } - } - - fn should_hash_part(part: &str, last_ext: Option<&str>) -> bool { - if part.is_empty() || part.len() > 30 { - // keep short due to windows path limit - return true; - } - let hash_context_specific = if let Some(last_ext) = last_ext { - // if the last part does not have a known extension, hash it in order to - // prevent collisions with a directory of the same name - !has_known_extension(part) || !part.ends_with(last_ext) - } else { - // if any non-ending path part has a known extension, hash it in order to - // prevent collisions where a filename has the same name as a directory name - has_known_extension(part) - }; - - // the hash symbol at the start designates a hash for the url part - hash_context_specific - || part.starts_with('#') - || has_forbidden_chars(part) - || last_ext.is_none() && FORBIDDEN_WINDOWS_NAMES.contains(part) - || part.ends_with('.') - } - - // get the base url - let port_separator = "_"; // make this shorter with just an underscore - let Some(mut base_parts) = base_url_to_filename_parts(url, port_separator) else { - return Err(UrlToFilenameConversionError { url: url.to_string() }); - }; - - if base_parts[0] == "https" { - base_parts.remove(0); - } else { - let scheme = base_parts.remove(0); - base_parts[0] = format!("{}_{}", scheme, base_parts[0]); - } - - // first, try to get the filename of the path - let path_segments = url_path_segments(url); - let mut parts = base_parts - .into_iter() - .chain(path_segments.map(|s| s.to_string())) - .collect::>(); - - // push the query parameter onto the last part - if let Some(query) = url.query() { - let last_part = parts.last_mut().unwrap(); - last_part.push('?'); - last_part.push_str(query); - } - - let mut has_hash = false; - let parts_len = parts.len(); - let parts = parts - .into_iter() - .enumerate() - .map(|(i, part)| { - let is_last = i == parts_len - 1; - let last_ext = if is_last { - Some(get_extension(url, content_type)) - } else { - None - }; - if should_hash_part(&part, last_ext) { - has_hash = true; - short_hash(&part, last_ext) - } else { - part - } - }) - .collect::>(); - - Ok(LocalCacheSubPath { has_hash, parts }) -} - -#[derive(Debug)] -struct LocalCacheManifest { - file_path: PathBuf, - data: RwLock, -} - -impl LocalCacheManifest { - pub fn new(file_path: PathBuf) -> Self { - Self::new_internal(file_path, false) - } - - pub fn new_for_lsp(file_path: PathBuf) -> Self { - Self::new_internal(file_path, true) - } - - fn new_internal(file_path: PathBuf, use_reverse_mapping: bool) -> Self { - let text = std::fs::read_to_string(&file_path).ok(); - Self { - data: RwLock::new(manifest::LocalCacheManifestData::new( - text.as_deref(), - use_reverse_mapping, - )), - file_path, - } - } - - pub fn insert_data( - &self, - sub_path: LocalCacheSubPath, - url: Url, - mut original_headers: HashMap, - ) { - fn should_keep_content_type_header( - url: &Url, - headers: &HashMap, - ) -> bool { - // only keep the location header if it can't be derived from the url - MediaType::from_specifier(url) - != MediaType::from_specifier_and_headers(url, Some(headers)) - } - - let mut headers_subset = IndexMap::new(); - - const HEADER_KEYS_TO_KEEP: [&str; 4] = [ - // keep alphabetical for cleanliness in the output - "content-type", - "location", - "x-deno-warning", - "x-typescript-types", - ]; - for key in HEADER_KEYS_TO_KEEP { - if key == "content-type" - && !should_keep_content_type_header(&url, &original_headers) - { - continue; - } - if let Some((k, v)) = original_headers.remove_entry(key) { - headers_subset.insert(k, v); - } - } - - let mut data = self.data.write(); - let add_module_entry = headers_subset.is_empty() - && !sub_path - .parts - .last() - .map(|s| s.starts_with('#')) - .unwrap_or(false); - let mut has_changed = if add_module_entry { - data.remove(&url, &sub_path) - } else { - let new_data = manifest::SerializedLocalCacheManifestDataModule { - headers: headers_subset, - }; - if data.get(&url) == Some(&new_data) { - false - } else { - data.insert(url.clone(), &sub_path, new_data); - true - } - }; - - if sub_path.has_hash { - let url_path_parts = url_path_segments(&url).collect::>(); - let base_url = { - let mut url = url.clone(); - url.set_path("/"); - url.set_query(None); - url.set_fragment(None); - url - }; - for (i, local_part) in sub_path.parts[1..sub_path.parts.len() - 1] - .iter() - .enumerate() - { - if local_part.starts_with('#') { - let mut url = base_url.clone(); - url.set_path(&format!("{}/", url_path_parts[..i + 1].join("/"))); - if data.add_directory(url, sub_path.parts[..i + 2].join("/")) { - has_changed = true; - } - } - } - } - - if has_changed { - // don't bother ensuring the directory here because it will - // eventually be created by files being added to the cache - let result = - atomic_write_file(&self.file_path, data.as_json(), CACHE_PERM); - if let Err(err) = result { - log::debug!("Failed saving local cache manifest: {:#}", err); - } - } - } - - pub fn get_metadata(&self, url: &Url) -> Option { - let data = self.data.read(); - match data.get(url) { - Some(module) => { - let headers = module - .headers - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect::>(); - let sub_path = if headers.contains_key("location") { - Cow::Borrowed(&self.file_path) - } else { - let sub_path = - url_to_local_sub_path(url, headers_content_type(&headers)).ok()?; - let folder_path = self.file_path.parent().unwrap(); - Cow::Owned(sub_path.as_path_from_root(folder_path)) - }; - - let Ok(metadata) = sub_path.metadata() else { - return None; - }; - - Some(CachedUrlMetadata { - headers, - url: url.to_string(), - time: metadata.modified().unwrap_or_else(|_| SystemTime::now()), - }) - } - None => { - let folder_path = self.file_path.parent().unwrap(); - let sub_path = url_to_local_sub_path(url, None).ok()?; - if sub_path - .parts - .last() - .map(|s| s.starts_with('#')) - .unwrap_or(false) - { - // only filenames without a hash are considered as in the cache - // when they don't have a metadata entry - return None; - } - let file_path = sub_path.as_path_from_root(folder_path); - if let Ok(metadata) = file_path.metadata() { - Some(CachedUrlMetadata { - headers: Default::default(), - url: url.to_string(), - time: metadata.modified().unwrap_or_else(|_| SystemTime::now()), - }) - } else { - None - } - } - } - } -} - -// This is in a separate module in order to enforce keeping -// the internal implementation private. -mod manifest { - use std::collections::HashMap; - use std::path::Path; - use std::path::PathBuf; - - use deno_core::serde_json; - use deno_core::url::Url; - use indexmap::IndexMap; - use serde::Deserialize; - use serde::Serialize; - - use super::url_to_local_sub_path; - use super::LocalCacheSubPath; - - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] - pub struct SerializedLocalCacheManifestDataModule { - #[serde( - default = "IndexMap::new", - skip_serializing_if = "IndexMap::is_empty" - )] - pub headers: IndexMap, - } - - impl SerializedLocalCacheManifestDataModule { - pub fn content_type_header(&self) -> Option<&str> { - self.headers.get("content-type").map(|s| s.as_str()) - } - } - - #[derive(Debug, Default, Clone, Serialize, Deserialize)] - struct SerializedLocalCacheManifestData { - #[serde( - default = "IndexMap::new", - skip_serializing_if = "IndexMap::is_empty" - )] - pub folders: IndexMap, - #[serde( - default = "IndexMap::new", - skip_serializing_if = "IndexMap::is_empty" - )] - pub modules: IndexMap, - } - - #[derive(Debug, Default, Clone)] - pub(super) struct LocalCacheManifestData { - serialized: SerializedLocalCacheManifestData, - // reverse mapping used in the lsp - reverse_mapping: Option>, - } - - impl LocalCacheManifestData { - pub fn new(maybe_text: Option<&str>, use_reverse_mapping: bool) -> Self { - let serialized: SerializedLocalCacheManifestData = maybe_text - .and_then(|text| match serde_json::from_str(text) { - Ok(data) => Some(data), - Err(err) => { - log::debug!("Failed deserializing local cache manifest: {:#}", err); - None - } - }) - .unwrap_or_default(); - let reverse_mapping = if use_reverse_mapping { - Some( - serialized - .modules - .iter() - .filter_map(|(url, module)| { - if module.headers.contains_key("location") { - return None; - } - url_to_local_sub_path(url, module.content_type_header()) - .ok() - .map(|local_path| { - let path = if cfg!(windows) { - PathBuf::from(local_path.parts.join("\\")) - } else { - PathBuf::from(local_path.parts.join("/")) - }; - (path, url.clone()) - }) - }) - .chain(serialized.folders.iter().map(|(url, local_path)| { - let path = if cfg!(windows) { - PathBuf::from(local_path.replace('/', "\\")) - } else { - PathBuf::from(local_path) - }; - (path, url.clone()) - })) - .collect::>(), - ) - } else { - None - }; - Self { - serialized, - reverse_mapping, - } - } - - pub fn get( - &self, - url: &Url, - ) -> Option<&SerializedLocalCacheManifestDataModule> { - self.serialized.modules.get(url) - } - - pub fn get_reverse_mapping(&self, path: &Path) -> Option { - debug_assert!(self.reverse_mapping.is_some()); // only call this if you're in the lsp - self - .reverse_mapping - .as_ref() - .and_then(|mapping| mapping.get(path)) - .cloned() - } - - pub fn add_directory(&mut self, url: Url, local_path: String) -> bool { - if let Some(current) = self.serialized.folders.get(&url) { - if *current == local_path { - return false; - } - } - - if let Some(reverse_mapping) = &mut self.reverse_mapping { - reverse_mapping.insert( - if cfg!(windows) { - PathBuf::from(local_path.replace('/', "\\")) - } else { - PathBuf::from(&local_path) - }, - url.clone(), - ); - } - - self.serialized.folders.insert(url, local_path); - true - } - - pub fn insert( - &mut self, - url: Url, - sub_path: &LocalCacheSubPath, - new_data: SerializedLocalCacheManifestDataModule, - ) { - if let Some(reverse_mapping) = &mut self.reverse_mapping { - reverse_mapping.insert(sub_path.as_relative_path(), url.clone()); - } - self.serialized.modules.insert(url, new_data); - } - - pub fn remove(&mut self, url: &Url, sub_path: &LocalCacheSubPath) -> bool { - if self.serialized.modules.remove(url).is_some() { - if let Some(reverse_mapping) = &mut self.reverse_mapping { - reverse_mapping.remove(&sub_path.as_relative_path()); - } - true - } else { - false - } - } - - pub fn as_json(&self) -> String { - serde_json::to_string_pretty(&self.serialized).unwrap() - } - } -} - -fn url_path_segments(url: &Url) -> impl Iterator { - url - .path() - .strip_prefix('/') - .unwrap_or(url.path()) - .split('/') -} - -#[cfg(test)] -mod test { - use super::*; - - use deno_core::serde_json::json; - use pretty_assertions::assert_eq; - use test_util::TempDir; - - #[test] - fn test_url_to_local_sub_path() { - run_test("https://deno.land/x/mod.ts", &[], "deno.land/x/mod.ts"); - run_test( - "http://deno.land/x/mod.ts", - &[], - // http gets added to the folder name, but not https - "http_deno.land/x/mod.ts", - ); - run_test( - // capital letter in filename - "https://deno.land/x/MOD.ts", - &[], - "deno.land/x/#mod_fa860.ts", - ); - run_test( - // query string - "https://deno.land/x/mod.ts?testing=1", - &[], - "deno.land/x/#mod_2eb80.ts", - ); - run_test( - // capital letter in directory - "https://deno.land/OTHER/mod.ts", - &[], - "deno.land/#other_1c55d/mod.ts", - ); - run_test( - // under max of 30 chars - "https://deno.land/x/012345678901234567890123456.js", - &[], - "deno.land/x/012345678901234567890123456.js", - ); - run_test( - // max 30 chars - "https://deno.land/x/0123456789012345678901234567.js", - &[], - "deno.land/x/#01234567890123456789_836de.js", - ); - run_test( - // forbidden char - "https://deno.land/x/mod's.js", - &[], - "deno.land/x/#mod_s_44fc8.js", - ); - run_test( - // no extension - "https://deno.land/x/mod", - &[("content-type", "application/typescript")], - "deno.land/x/#mod_e55cf.ts", - ); - run_test( - // known extension in directory is not allowed - // because it could conflict with a file of the same name - "https://deno.land/x/mod.js/mod.js", - &[], - "deno.land/x/#mod.js_59c58/mod.js", - ); - run_test( - // slash slash in path - "http://localhost//mod.js", - &[], - "http_localhost/#e3b0c44/mod.js", - ); - run_test( - // headers same extension - "https://deno.land/x/mod.ts", - &[("content-type", "application/typescript")], - "deno.land/x/mod.ts", - ); - run_test( - // headers different extension... We hash this because - // if someone deletes the manifest file, then we don't want - // https://deno.land/x/mod.ts to resolve as a typescript file - "https://deno.land/x/mod.ts", - &[("content-type", "application/javascript")], - "deno.land/x/#mod.ts_e8c36.js", - ); - run_test( - // not allowed windows folder name - "https://deno.land/x/con/con.ts", - &[], - "deno.land/x/#con_1143d/con.ts", - ); - run_test( - // disallow ending a directory with a period - // https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file - "https://deno.land/x/test./main.ts", - &[], - "deno.land/x/#test._4ee3d/main.ts", - ); - - #[track_caller] - fn run_test(url: &str, headers: &[(&str, &str)], expected: &str) { - let url = Url::parse(url).unwrap(); - let headers = headers - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - let result = - url_to_local_sub_path(&url, headers_content_type(&headers)).unwrap(); - let parts = result.parts.join("/"); - assert_eq!(parts, expected); - assert_eq!( - result.parts.iter().any(|p| p.starts_with('#')), - result.has_hash - ) - } - } - - #[test] - fn test_local_global_cache() { - let temp_dir = TempDir::new(); - let global_cache_path = temp_dir.path().join("global"); - let local_cache_path = temp_dir.path().join("local"); - let global_cache = - Arc::new(GlobalHttpCache::new(global_cache_path.to_path_buf())); - let local_cache = - LocalHttpCache::new(local_cache_path.to_path_buf(), global_cache.clone()); - - let manifest_file = local_cache_path.join("manifest.json"); - // mapped url - { - let url = Url::parse("https://deno.land/x/mod.ts").unwrap(); - let content = "export const test = 5;"; - global_cache - .set( - &url, - HashMap::from([( - "content-type".to_string(), - "application/typescript".to_string(), - )]), - content.as_bytes(), - ) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - // won't have any headers because the content-type is derivable from the url - assert_eq!(metadata.headers, HashMap::new()); - assert_eq!(metadata.url, url.to_string()); - // no manifest file yet - assert!(!manifest_file.exists()); - - // now try deleting the global cache and we should still be able to load it - global_cache_path.remove_dir_all(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - } - - // file that's directly mappable to a url - { - let content = "export const a = 1;"; - local_cache_path - .join("deno.land") - .join("main.js") - .write(content); - - // now we should be able to read this file because it's directly mappable to a url - let url = Url::parse("https://deno.land/main.js").unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - assert_eq!(metadata.headers, HashMap::new()); - assert_eq!(metadata.url, url.to_string()); - } - - // now try a file with a different content-type header - { - let url = - Url::parse("https://deno.land/x/different_content_type.ts").unwrap(); - let content = "export const test = 5;"; - global_cache - .set( - &url, - HashMap::from([( - "content-type".to_string(), - "application/javascript".to_string(), - )]), - content.as_bytes(), - ) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - assert_eq!( - metadata.headers, - HashMap::from([( - "content-type".to_string(), - "application/javascript".to_string(), - )]) - ); - assert_eq!(metadata.url, url.to_string()); - assert_eq!( - manifest_file.read_json_value(), - json!({ - "modules": { - "https://deno.land/x/different_content_type.ts": { - "headers": { - "content-type": "application/javascript" - } - } - } - }) - ); - // delete the manifest file - manifest_file.remove_file(); - - // Now try resolving the key again and the content type should still be application/javascript. - // This is maintained because we hash the filename when the headers don't match the extension. - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - assert_eq!( - metadata.headers, - HashMap::from([( - "content-type".to_string(), - "application/javascript".to_string(), - )]) - ); - } - - // reset the local cache - local_cache_path.remove_dir_all(); - let local_cache = - LocalHttpCache::new(local_cache_path.to_path_buf(), global_cache.clone()); - - // now try caching a file with many headers - { - let url = Url::parse("https://deno.land/x/my_file.ts").unwrap(); - let content = "export const test = 5;"; - global_cache - .set( - &url, - HashMap::from([ - ( - "content-type".to_string(), - "application/typescript".to_string(), - ), - ("x-typescript-types".to_string(), "./types.d.ts".to_string()), - ("x-deno-warning".to_string(), "Stop right now.".to_string()), - ( - "x-other-header".to_string(), - "Thank you very much.".to_string(), - ), - ]), - content.as_bytes(), - ) - .unwrap(); - let check_output = |local_cache: &LocalHttpCache| { - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - assert_eq!( - metadata.headers, - HashMap::from([ - ("x-typescript-types".to_string(), "./types.d.ts".to_string(),), - ("x-deno-warning".to_string(), "Stop right now.".to_string(),) - ]) - ); - assert_eq!(metadata.url, url.to_string()); - assert_eq!( - manifest_file.read_json_value(), - json!({ - "modules": { - "https://deno.land/x/my_file.ts": { - "headers": { - "x-deno-warning": "Stop right now.", - "x-typescript-types": "./types.d.ts" - } - } - } - }) - ); - }; - check_output(&local_cache); - // now ensure it's the same when re-creating the cache - check_output(&LocalHttpCache::new( - local_cache_path.to_path_buf(), - global_cache.clone(), - )); - } - - // reset the local cache - local_cache_path.remove_dir_all(); - let local_cache = - LocalHttpCache::new(local_cache_path.to_path_buf(), global_cache.clone()); - - // try a file that can't be mapped to the file system - { - { - let url = - Url::parse("https://deno.land/INVALID/Module.ts?dev").unwrap(); - let content = "export const test = 5;"; - global_cache - .set(&url, HashMap::new(), content.as_bytes()) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - // won't have any headers because the content-type is derivable from the url - assert_eq!(metadata.headers, HashMap::new()); - assert_eq!(metadata.url, url.to_string()); - } - - // now try a file in the same directory, but that maps to the local filesystem - { - let url = Url::parse("https://deno.land/INVALID/module2.ts").unwrap(); - let content = "export const test = 4;"; - global_cache - .set(&url, HashMap::new(), content.as_bytes()) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - assert!(local_cache_path - .join("deno.land/#invalid_1ee01/module2.ts") - .exists()); - - // ensure we can still read this file with a new local cache - let local_cache = LocalHttpCache::new( - local_cache_path.to_path_buf(), - global_cache.clone(), - ); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - } - - assert_eq!( - manifest_file.read_json_value(), - json!({ - "modules": { - "https://deno.land/INVALID/Module.ts?dev": { - } - }, - "folders": { - "https://deno.land/INVALID/": "deno.land/#invalid_1ee01", - } - }) - ); - } - - // reset the local cache - local_cache_path.remove_dir_all(); - let local_cache = - LocalHttpCache::new(local_cache_path.to_path_buf(), global_cache.clone()); - - // now try a redirect - { - let url = Url::parse("https://deno.land/redirect.ts").unwrap(); - global_cache - .set( - &url, - HashMap::from([("location".to_string(), "./x/mod.ts".to_string())]), - "Redirecting to other url...".as_bytes(), - ) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - let metadata = local_cache.read_metadata(&key).unwrap().unwrap(); - assert_eq!( - metadata.headers, - HashMap::from([("location".to_string(), "./x/mod.ts".to_string())]) - ); - assert_eq!(metadata.url, url.to_string()); - assert_eq!( - manifest_file.read_json_value(), - json!({ - "modules": { - "https://deno.land/redirect.ts": { - "headers": { - "location": "./x/mod.ts" - } - } - } - }) - ); - } - } - - #[test] - fn test_lsp_local_cache() { - let temp_dir = TempDir::new(); - let global_cache_path = temp_dir.path().join("global"); - let local_cache_path = temp_dir.path().join("local"); - let global_cache = - Arc::new(GlobalHttpCache::new(global_cache_path.to_path_buf())); - let local_cache = LocalLspHttpCache::new( - local_cache_path.to_path_buf(), - global_cache.clone(), - ); - - // mapped url - { - let url = Url::parse("https://deno.land/x/mod.ts").unwrap(); - let content = "export const test = 5;"; - global_cache - .set( - &url, - HashMap::from([( - "content-type".to_string(), - "application/typescript".to_string(), - )]), - content.as_bytes(), - ) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - - // check getting the file url works - let file_url = local_cache.get_file_url(&url); - let expected = local_cache_path - .uri_dir() - .join("deno.land/x/mod.ts") - .unwrap(); - assert_eq!(file_url, Some(expected)); - - // get the reverse mapping - let mapping = local_cache.get_remote_url( - local_cache_path - .join("deno.land") - .join("x") - .join("mod.ts") - .as_path(), - ); - assert_eq!(mapping.as_ref(), Some(&url)); - } - - // now try a file with a different content-type header - { - let url = - Url::parse("https://deno.land/x/different_content_type.ts").unwrap(); - let content = "export const test = 5;"; - global_cache - .set( - &url, - HashMap::from([( - "content-type".to_string(), - "application/javascript".to_string(), - )]), - content.as_bytes(), - ) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8(local_cache.read_file_bytes(&key).unwrap().unwrap()) - .unwrap(), - content - ); - - let file_url = local_cache.get_file_url(&url).unwrap(); - let path = file_url.to_file_path().unwrap(); - assert!(path.exists()); - let mapping = local_cache.get_remote_url(&path); - assert_eq!(mapping.as_ref(), Some(&url)); - } - - // try http specifiers that can't be mapped to the file system - { - let urls = [ - "http://deno.land/INVALID/Module.ts?dev", - "http://deno.land/INVALID/SubDir/Module.ts?dev", - ]; - for url in urls { - let url = Url::parse(url).unwrap(); - let content = "export const test = 5;"; - global_cache - .set(&url, HashMap::new(), content.as_bytes()) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - - let file_url = local_cache.get_file_url(&url).unwrap(); - let path = file_url.to_file_path().unwrap(); - assert!(path.exists()); - let mapping = local_cache.get_remote_url(&path); - assert_eq!(mapping.as_ref(), Some(&url)); - } - - // now try a files in the same and sub directories, that maps to the local filesystem - let urls = [ - "http://deno.land/INVALID/module2.ts", - "http://deno.land/INVALID/SubDir/module3.ts", - "http://deno.land/INVALID/SubDir/sub_dir/module4.ts", - ]; - for url in urls { - let url = Url::parse(url).unwrap(); - let content = "export const test = 4;"; - global_cache - .set(&url, HashMap::new(), content.as_bytes()) - .unwrap(); - let key = local_cache.cache_item_key(&url).unwrap(); - assert_eq!( - String::from_utf8( - local_cache.read_file_bytes(&key).unwrap().unwrap() - ) - .unwrap(), - content - ); - let file_url = local_cache.get_file_url(&url).unwrap(); - let path = file_url.to_file_path().unwrap(); - assert!(path.exists()); - let mapping = local_cache.get_remote_url(&path); - assert_eq!(mapping.as_ref(), Some(&url)); - - // ensure we can still get this file with a new local cache - let local_cache = LocalLspHttpCache::new( - local_cache_path.to_path_buf(), - global_cache.clone(), - ); - let file_url = local_cache.get_file_url(&url).unwrap(); - let path = file_url.to_file_path().unwrap(); - assert!(path.exists()); - let mapping = local_cache.get_remote_url(&path); - assert_eq!(mapping.as_ref(), Some(&url)); - } - } - } -} diff --git a/cli/cache/http_cache/mod.rs b/cli/cache/http_cache/mod.rs deleted file mode 100644 index 8d09b0995f..0000000000 --- a/cli/cache/http_cache/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use deno_core::error::AnyError; -use deno_core::serde::Deserialize; -use deno_core::serde::Serialize; -use deno_core::url::Url; -use std::path::PathBuf; -use std::time::SystemTime; - -use crate::http_util::HeadersMap; - -mod common; -mod global; -mod local; - -pub use global::url_to_filename; -pub use global::GlobalHttpCache; -pub use local::LocalHttpCache; -pub use local::LocalLspHttpCache; - -/// Cached metadata about a url. -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub struct CachedUrlMetadata { - pub headers: HeadersMap, - pub url: String, - #[serde(default = "SystemTime::now", rename = "now")] - pub time: SystemTime, -} - -impl CachedUrlMetadata { - pub fn is_redirect(&self) -> bool { - self.headers.contains_key("location") - } -} - -/// Computed cache key, which can help reduce the work of computing the cache key multiple times. -pub struct HttpCacheItemKey<'a> { - // The key is specific to the implementation of HttpCache, - // so keep these private to the module. For example, the - // fact that these may be stored in a file is an implementation - // detail. - #[cfg(debug_assertions)] - pub(super) is_local_key: bool, - pub(super) url: &'a Url, - /// This will be set all the time for the global cache, but it - /// won't ever be set for the local cache because that also needs - /// header information to determine the final path. - pub(super) file_path: Option, -} - -pub trait HttpCache: Send + Sync + std::fmt::Debug { - /// A pre-computed key for looking up items in the cache. - fn cache_item_key<'a>( - &self, - url: &'a Url, - ) -> Result, AnyError>; - - fn contains(&self, url: &Url) -> bool; - fn set( - &self, - url: &Url, - headers: HeadersMap, - content: &[u8], - ) -> Result<(), AnyError>; - fn read_modified_time( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError>; - fn read_file_bytes( - &self, - key: &HttpCacheItemKey, - ) -> Result>, AnyError>; - fn read_metadata( - &self, - key: &HttpCacheItemKey, - ) -> Result, AnyError>; -} diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 2427133d12..f75761521a 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -2,6 +2,7 @@ use crate::errors::get_error_class_name; use crate::file_fetcher::FileFetcher; +use crate::util::fs::atomic_write_file; use deno_core::futures; use deno_core::futures::FutureExt; @@ -12,8 +13,10 @@ use deno_graph::source::LoadResponse; use deno_graph::source::Loader; use deno_runtime::permissions::PermissionsContainer; use std::collections::HashMap; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use std::time::SystemTime; mod cache_db; mod caches; @@ -22,7 +25,6 @@ mod common; mod deno_dir; mod disk_cache; mod emit; -mod http_cache; mod incremental; mod node; mod parsed_source; @@ -34,11 +36,6 @@ pub use deno_dir::DenoDir; pub use deno_dir::DenoDirProvider; pub use disk_cache::DiskCache; pub use emit::EmitCache; -pub use http_cache::CachedUrlMetadata; -pub use http_cache::GlobalHttpCache; -pub use http_cache::HttpCache; -pub use http_cache::LocalHttpCache; -pub use http_cache::LocalLspHttpCache; pub use incremental::IncrementalCache; pub use node::NodeAnalysisCache; pub use parsed_source::ParsedSourceCache; @@ -46,6 +43,52 @@ pub use parsed_source::ParsedSourceCache; /// Permissions used to save a file in the disk caches. pub const CACHE_PERM: u32 = 0o644; +#[derive(Debug, Clone)] +pub struct RealDenoCacheEnv; + +impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv { + fn read_file_bytes(&self, path: &Path) -> std::io::Result>> { + match std::fs::read(path) { + Ok(s) => Ok(Some(s)), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + fn atomic_write_file( + &self, + path: &Path, + bytes: &[u8], + ) -> std::io::Result<()> { + atomic_write_file(path, bytes, CACHE_PERM) + } + + fn modified(&self, path: &Path) -> std::io::Result> { + match std::fs::metadata(path) { + Ok(metadata) => Ok(Some( + metadata.modified().unwrap_or_else(|_| SystemTime::now()), + )), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(err) => Err(err), + } + } + + fn is_file(&self, path: &Path) -> bool { + path.is_file() + } + + fn time_now(&self) -> SystemTime { + SystemTime::now() + } +} + +pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache; +pub type LocalHttpCache = deno_cache_dir::LocalHttpCache; +pub type LocalLspHttpCache = + deno_cache_dir::LocalLspHttpCache; +pub use deno_cache_dir::CachedUrlMetadata; +pub use deno_cache_dir::HttpCache; + /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { diff --git a/cli/factory.rs b/cli/factory.rs index e9a50aadaf..9c553620df 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -241,6 +241,7 @@ impl CliFactory { self.services.global_http_cache.get_or_try_init(|| { Ok(Arc::new(GlobalHttpCache::new( self.deno_dir()?.deps_folder_path(), + crate::cache::RealDenoCacheEnv, ))) }) } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 864c59cb59..d79bb4368e 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -696,6 +696,7 @@ async fn fetch_once<'a>( #[cfg(test)] mod tests { use crate::cache::GlobalHttpCache; + use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClient; use crate::version; @@ -727,7 +728,7 @@ mod tests { let location = temp_dir.path().join("deps").to_path_buf(); let blob_store: Arc = Default::default(); let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), cache_setting, true, Arc::new(HttpClient::new(None, None)), @@ -1173,7 +1174,10 @@ mod tests { // invocation and indicates to "cache bust". let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )), CacheSetting::ReloadAll, true, Arc::new(HttpClient::new(None, None)), @@ -1202,7 +1206,10 @@ mod tests { let file_modified_01 = { let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location.clone())), + Arc::new(GlobalHttpCache::new( + location.clone(), + crate::cache::RealDenoCacheEnv, + )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1231,7 +1238,10 @@ mod tests { let file_modified_02 = { let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1362,7 +1372,10 @@ mod tests { let metadata_file_modified_01 = { let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location.clone())), + Arc::new(GlobalHttpCache::new( + location.clone(), + crate::cache::RealDenoCacheEnv, + )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1394,7 +1407,10 @@ mod tests { let metadata_file_modified_02 = { let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1496,7 +1512,10 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )), CacheSetting::Use, false, Arc::new(HttpClient::new(None, None)), @@ -1521,7 +1540,7 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher_01 = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location.clone())), + Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), CacheSetting::Only, true, Arc::new(HttpClient::new(None, None)), @@ -1529,7 +1548,7 @@ mod tests { None, ); let file_fetcher_02 = FileFetcher::new( - Arc::new(GlobalHttpCache::new(location)), + Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index a895239a64..60244f2e40 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -520,14 +520,20 @@ mod tests { source_fixtures: &[(&str, &str)], location: &Path, ) -> Documents { - let cache = Arc::new(GlobalHttpCache::new(location.to_path_buf())); + let cache = Arc::new(GlobalHttpCache::new( + location.to_path_buf(), + crate::cache::RealDenoCacheEnv, + )); let mut documents = Documents::new(cache); for (specifier, source, version, language_id) in fixtures { let specifier = resolve_url(specifier).expect("failed to create specifier"); documents.open(specifier, *version, *language_id, (*source).into()); } - let http_cache = GlobalHttpCache::new(location.to_path_buf()); + let http_cache = GlobalHttpCache::new( + location.to_path_buf(), + crate::cache::RealDenoCacheEnv, + ); for (specifier, source) in source_fixtures { let specifier = resolve_url(specifier).expect("failed to create specifier"); diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs index 88c4c91cb6..4e24673f3b 100644 --- a/cli/lsp/diagnostics.rs +++ b/cli/lsp/diagnostics.rs @@ -1331,6 +1331,7 @@ fn generate_deno_diagnostics( mod tests { use super::*; use crate::cache::GlobalHttpCache; + use crate::cache::RealDenoCacheEnv; use crate::lsp::config::ConfigSnapshot; use crate::lsp::config::Settings; use crate::lsp::config::SpecifierSettings; @@ -1349,7 +1350,10 @@ mod tests { location: &Path, maybe_import_map: Option<(&str, &str)>, ) -> StateSnapshot { - let cache = Arc::new(GlobalHttpCache::new(location.to_path_buf())); + let cache = Arc::new(GlobalHttpCache::new( + location.to_path_buf(), + RealDenoCacheEnv, + )); let mut documents = Documents::new(cache); for (specifier, source, version, language_id) in fixtures { let specifier = @@ -1374,7 +1378,7 @@ mod tests { maybe_import_map, assets: Default::default(), cache_metadata: cache::CacheMetadata::new(Arc::new( - GlobalHttpCache::new(location.to_path_buf()), + GlobalHttpCache::new(location.to_path_buf(), RealDenoCacheEnv), )), maybe_node_resolver: None, maybe_npm_resolver: None, @@ -1424,7 +1428,8 @@ let c: number = "a"; None, ); let snapshot = Arc::new(snapshot); - let cache = Arc::new(GlobalHttpCache::new(cache_location)); + let cache = + Arc::new(GlobalHttpCache::new(cache_location, RealDenoCacheEnv)); let ts_server = TsServer::new(Default::default(), cache); // test enabled @@ -1517,7 +1522,8 @@ let c: number = "a"; None, ); let snapshot = Arc::new(snapshot); - let cache = Arc::new(GlobalHttpCache::new(cache_location)); + let cache = + Arc::new(GlobalHttpCache::new(cache_location, RealDenoCacheEnv)); let ts_server = TsServer::new(Default::default(), cache); let config = mock_config(); diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index 7fdce99234..439cf547d6 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -1851,6 +1851,7 @@ fn sort_and_remove_non_leaf_dirs(mut dirs: Vec) -> Vec { #[cfg(test)] mod tests { use crate::cache::GlobalHttpCache; + use crate::cache::RealDenoCacheEnv; use crate::npm::NpmResolution; use super::*; @@ -1861,7 +1862,10 @@ mod tests { fn setup(temp_dir: &TempDir) -> (Documents, PathRef) { let location = temp_dir.path().join("deps"); - let cache = Arc::new(GlobalHttpCache::new(location.to_path_buf())); + let cache = Arc::new(GlobalHttpCache::new( + location.to_path_buf(), + RealDenoCacheEnv, + )); let documents = Documents::new(cache); (documents, location) } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 7a13e23967..5b9ba94f1f 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -555,7 +555,10 @@ impl Inner { http_client.clone(), ); let location = dir.deps_folder_path(); - let deps_http_cache = Arc::new(GlobalHttpCache::new(location)); + let deps_http_cache = Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )); let documents = Documents::new(deps_http_cache.clone()); let cache_metadata = cache::CacheMetadata::new(deps_http_cache.clone()); let performance = Arc::new(Performance::default()); @@ -904,7 +907,10 @@ impl Inner { ); self.module_registries_location = module_registries_location; // update the cache path - let global_cache = Arc::new(GlobalHttpCache::new(dir.deps_folder_path())); + let global_cache = Arc::new(GlobalHttpCache::new( + dir.deps_folder_path(), + crate::cache::RealDenoCacheEnv, + )); let maybe_local_cache = self.config.maybe_vendor_dir_path().map(|local_path| { Arc::new(LocalLspHttpCache::new(local_path, global_cache.clone())) diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 0219653bd6..186db50b8c 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -422,7 +422,10 @@ pub struct ModuleRegistry { impl ModuleRegistry { pub fn new(location: PathBuf, http_client: Arc) -> Self { // the http cache should always be the global one for registry completions - let http_cache = Arc::new(GlobalHttpCache::new(location)); + let http_cache = Arc::new(GlobalHttpCache::new( + location, + crate::cache::RealDenoCacheEnv, + )); let mut file_fetcher = FileFetcher::new( http_cache.clone(), CacheSetting::RespectHeaders, diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 3538665130..75ed8ebe38 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -3915,6 +3915,7 @@ mod tests { use super::*; use crate::cache::GlobalHttpCache; use crate::cache::HttpCache; + use crate::cache::RealDenoCacheEnv; use crate::http_util::HeadersMap; use crate::lsp::cache::CacheMetadata; use crate::lsp::config::WorkspaceSettings; @@ -3931,7 +3932,10 @@ mod tests { fixtures: &[(&str, &str, i32, LanguageId)], location: &Path, ) -> StateSnapshot { - let cache = Arc::new(GlobalHttpCache::new(location.to_path_buf())); + let cache = Arc::new(GlobalHttpCache::new( + location.to_path_buf(), + RealDenoCacheEnv, + )); let mut documents = Documents::new(cache.clone()); for (specifier, source, version, language_id) in fixtures { let specifier = @@ -3960,7 +3964,8 @@ mod tests { sources: &[(&str, &str, i32, LanguageId)], ) -> (JsRuntime, Arc, PathBuf) { let location = temp_dir.path().join("deps").to_path_buf(); - let cache = Arc::new(GlobalHttpCache::new(location.clone())); + let cache = + Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)); let state_snapshot = Arc::new(mock_state_snapshot(sources, &location)); let mut runtime = js_runtime(Default::default(), cache); start(&mut runtime, debug).unwrap(); @@ -4440,7 +4445,7 @@ mod tests { LanguageId::TypeScript, )], ); - let cache = Arc::new(GlobalHttpCache::new(location)); + let cache = Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)); let specifier_dep = resolve_url("https://deno.land/x/example/a.ts").unwrap(); cache