mirror of
https://github.com/denoland/deno.git
synced 2024-12-24 08:09:08 -05:00
feat: Add --no-remote, rename --no-fetch to --cached-only (#3417)
This commit is contained in:
parent
ee013102ff
commit
91da410fc3
6 changed files with 148 additions and 55 deletions
|
@ -74,7 +74,8 @@ pub struct SourceFileFetcher {
|
|||
source_file_cache: SourceFileCache,
|
||||
cache_blacklist: Vec<String>,
|
||||
use_disk_cache: bool,
|
||||
no_remote_fetch: bool,
|
||||
no_remote: bool,
|
||||
cached_only: bool,
|
||||
}
|
||||
|
||||
impl SourceFileFetcher {
|
||||
|
@ -83,7 +84,8 @@ impl SourceFileFetcher {
|
|||
progress: Progress,
|
||||
use_disk_cache: bool,
|
||||
cache_blacklist: Vec<String>,
|
||||
no_remote_fetch: bool,
|
||||
no_remote: bool,
|
||||
cached_only: bool,
|
||||
) -> std::io::Result<Self> {
|
||||
let file_fetcher = Self {
|
||||
deps_cache,
|
||||
|
@ -91,7 +93,8 @@ impl SourceFileFetcher {
|
|||
source_file_cache: SourceFileCache::default(),
|
||||
cache_blacklist,
|
||||
use_disk_cache,
|
||||
no_remote_fetch,
|
||||
no_remote,
|
||||
cached_only,
|
||||
};
|
||||
|
||||
Ok(file_fetcher)
|
||||
|
@ -123,10 +126,10 @@ impl SourceFileFetcher {
|
|||
|
||||
// If file is not in memory cache check if it can be found
|
||||
// in local cache - which effectively means trying to fetch
|
||||
// using "--no-fetch" flag. We can safely block on this
|
||||
// using "--cached-only" flag. We can safely block on this
|
||||
// future, because it doesn't do any asynchronous action
|
||||
// it that path.
|
||||
let fut = self.get_source_file_async(specifier.as_url(), true, true);
|
||||
let fut = self.get_source_file_async(specifier.as_url(), true, false, true);
|
||||
|
||||
futures::executor::block_on(fut).ok()
|
||||
}
|
||||
|
@ -152,21 +155,31 @@ impl SourceFileFetcher {
|
|||
.get_source_file_async(
|
||||
&module_url,
|
||||
self.use_disk_cache,
|
||||
self.no_remote_fetch,
|
||||
self.no_remote,
|
||||
self.cached_only,
|
||||
)
|
||||
.then(move |result| {
|
||||
let mut out = match result.map_err(|err| {
|
||||
if err.kind() == ErrorKind::NotFound {
|
||||
let msg = if let Some(referrer) = maybe_referrer {
|
||||
format!(
|
||||
"Cannot resolve module \"{}\" from \"{}\"",
|
||||
module_url.to_string(),
|
||||
referrer
|
||||
)
|
||||
let err_kind = err.kind();
|
||||
let referrer_suffix = if let Some(referrer) = maybe_referrer {
|
||||
format!(" from \"{}\"", referrer)
|
||||
} else {
|
||||
format!("Cannot resolve module \"{}\"", module_url.to_string())
|
||||
"".to_owned()
|
||||
};
|
||||
if err_kind == ErrorKind::NotFound {
|
||||
let msg = format!(
|
||||
"Cannot resolve module \"{}\"{}",
|
||||
module_url.to_string(),
|
||||
referrer_suffix
|
||||
);
|
||||
DenoError::new(ErrorKind::NotFound, msg).into()
|
||||
} else if err_kind == ErrorKind::PermissionDenied {
|
||||
let msg = format!(
|
||||
"Cannot find module \"{}\"{} in cache, --cached-only is specified",
|
||||
module_url.to_string(),
|
||||
referrer_suffix
|
||||
);
|
||||
DenoError::new(ErrorKind::PermissionDenied, msg).into()
|
||||
} else {
|
||||
err
|
||||
}
|
||||
|
@ -196,13 +209,16 @@ impl SourceFileFetcher {
|
|||
///
|
||||
/// If `use_disk_cache` is true then remote files are fetched from disk cache.
|
||||
///
|
||||
/// If `no_remote_fetch` is true then if remote file is not found it disk
|
||||
/// cache this method will fail.
|
||||
/// If `no_remote` is true then this method will fail for remote files.
|
||||
///
|
||||
/// If `cached_only` is true then this method will fail for remote files
|
||||
/// not already cached.
|
||||
fn get_source_file_async(
|
||||
self: &Self,
|
||||
module_url: &Url,
|
||||
use_disk_cache: bool,
|
||||
no_remote_fetch: bool,
|
||||
no_remote: bool,
|
||||
cached_only: bool,
|
||||
) -> impl Future<Output = Result<SourceFile, ErrBox>> {
|
||||
let url_scheme = module_url.scheme();
|
||||
let is_local_file = url_scheme == "file";
|
||||
|
@ -224,11 +240,25 @@ impl SourceFileFetcher {
|
|||
}
|
||||
}
|
||||
|
||||
// The file is remote, fail if `no_remote` is true.
|
||||
if no_remote {
|
||||
return Either::Left(futures::future::err(
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!(
|
||||
"Not allowed to get remote file '{}'",
|
||||
module_url.to_string()
|
||||
),
|
||||
)
|
||||
.into(),
|
||||
));
|
||||
}
|
||||
|
||||
// Fetch remote file and cache on-disk for subsequent access
|
||||
Either::Right(self.fetch_remote_source_async(
|
||||
&module_url,
|
||||
use_disk_cache,
|
||||
no_remote_fetch,
|
||||
cached_only,
|
||||
10,
|
||||
))
|
||||
}
|
||||
|
@ -327,7 +357,7 @@ impl SourceFileFetcher {
|
|||
self: &Self,
|
||||
module_url: &Url,
|
||||
use_disk_cache: bool,
|
||||
no_remote_fetch: bool,
|
||||
cached_only: bool,
|
||||
redirect_limit: i64,
|
||||
) -> Pin<Box<SourceFileFuture>> {
|
||||
if redirect_limit < 0 {
|
||||
|
@ -352,11 +382,11 @@ impl SourceFileFetcher {
|
|||
}
|
||||
|
||||
// If file wasn't found in cache check if we can fetch it
|
||||
if no_remote_fetch {
|
||||
if cached_only {
|
||||
// We can't fetch remote file - bail out
|
||||
return futures::future::err(
|
||||
std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
format!(
|
||||
"cannot find remote file '{}' in cache",
|
||||
module_url.to_string()
|
||||
|
@ -391,7 +421,7 @@ impl SourceFileFetcher {
|
|||
Either::Left(dir.fetch_remote_source_async(
|
||||
&new_module_url,
|
||||
use_disk_cache,
|
||||
no_remote_fetch,
|
||||
cached_only,
|
||||
redirect_limit - 1,
|
||||
))
|
||||
}
|
||||
|
@ -690,6 +720,7 @@ mod tests {
|
|||
true,
|
||||
vec![],
|
||||
false,
|
||||
false,
|
||||
)
|
||||
.expect("setup fail")
|
||||
}
|
||||
|
@ -843,7 +874,7 @@ mod tests {
|
|||
let headers_file_name_3 = headers_file_name.clone();
|
||||
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&module_url, true, false)
|
||||
.get_source_file_async(&module_url, true, false, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
let r = result.unwrap();
|
||||
|
@ -860,7 +891,7 @@ mod tests {
|
|||
&headers_file_name_1,
|
||||
"{ \"mime_type\": \"text/javascript\" }",
|
||||
);
|
||||
fetcher_1.get_source_file_async(&module_url, true, false)
|
||||
fetcher_1.get_source_file_async(&module_url, true, false, false)
|
||||
})
|
||||
.then(move |result2| {
|
||||
assert!(result2.is_ok());
|
||||
|
@ -886,7 +917,7 @@ mod tests {
|
|||
Some("application/json".to_owned()),
|
||||
None,
|
||||
);
|
||||
fetcher_2.get_source_file_async(&module_url_1, true, false)
|
||||
fetcher_2.get_source_file_async(&module_url_1, true, false, false)
|
||||
})
|
||||
.then(move |result3| {
|
||||
assert!(result3.is_ok());
|
||||
|
@ -905,7 +936,7 @@ mod tests {
|
|||
// let's create fresh instance of DenoDir (simulating another freshh Deno process)
|
||||
// and don't use cache
|
||||
let fetcher = setup_file_fetcher(temp_dir.path());
|
||||
fetcher.get_source_file_async(&module_url_2, false, false)
|
||||
fetcher.get_source_file_async(&module_url_2, false, false, false)
|
||||
})
|
||||
.then(move |result4| {
|
||||
assert!(result4.is_ok());
|
||||
|
@ -940,7 +971,7 @@ mod tests {
|
|||
);
|
||||
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&module_url, true, false)
|
||||
.get_source_file_async(&module_url, true, false, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
let r = result.unwrap();
|
||||
|
@ -962,7 +993,7 @@ mod tests {
|
|||
Some("text/typescript".to_owned()),
|
||||
None,
|
||||
);
|
||||
fetcher.get_source_file_async(&module_url, true, false)
|
||||
fetcher.get_source_file_async(&module_url, true, false, false)
|
||||
})
|
||||
.then(move |result2| {
|
||||
assert!(result2.is_ok());
|
||||
|
@ -977,7 +1008,7 @@ mod tests {
|
|||
// let's create fresh instance of DenoDir (simulating another freshh Deno process)
|
||||
// and don't use cache
|
||||
let fetcher = setup_file_fetcher(temp_dir.path());
|
||||
fetcher.get_source_file_async(&module_url_1, false, false)
|
||||
fetcher.get_source_file_async(&module_url_1, false, false, false)
|
||||
})
|
||||
.then(move |result3| {
|
||||
assert!(result3.is_ok());
|
||||
|
@ -1077,7 +1108,7 @@ mod tests {
|
|||
|
||||
// Test basic follow and headers recording
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&redirect_module_url, true, false)
|
||||
.get_source_file_async(&redirect_module_url, true, false, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
let mod_meta = result.unwrap();
|
||||
|
@ -1138,7 +1169,7 @@ mod tests {
|
|||
|
||||
// Test double redirects and headers recording
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&double_redirect_url, true, false)
|
||||
.get_source_file_async(&double_redirect_url, true, false, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
let mod_meta = result.unwrap();
|
||||
|
@ -1196,7 +1227,7 @@ mod tests {
|
|||
|
||||
// Test that redirect target is not downloaded twice for different redirect source.
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&double_redirect_url, true, false)
|
||||
.get_source_file_async(&double_redirect_url, true, false, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
let result = fs::File::open(&target_path);
|
||||
|
@ -1210,7 +1241,7 @@ mod tests {
|
|||
// shouldn't be downloaded again. It can be verified using source header file creation
|
||||
// timestamp (should be the same as after first `get_source_file`)
|
||||
fetcher
|
||||
.get_source_file_async(&redirect_url, true, false)
|
||||
.get_source_file_async(&redirect_url, true, false, false)
|
||||
.map(move |r| (r, file_modified))
|
||||
})
|
||||
.then(move |(result, file_modified)| {
|
||||
|
@ -1257,7 +1288,27 @@ mod tests {
|
|||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_source_code_no_fetch() {
|
||||
fn test_get_source_no_remote() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
let (_temp_dir, fetcher) = test_setup();
|
||||
let module_url =
|
||||
Url::parse("http://localhost:4545/tests/002_hello.ts").unwrap();
|
||||
// Remote modules are not allowed
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&module_url, true, true, false)
|
||||
.then(move |result| {
|
||||
assert!(result.is_err());
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.kind(), ErrorKind::NotFound);
|
||||
futures::future::ok(())
|
||||
});
|
||||
|
||||
tokio_util::run(fut);
|
||||
drop(http_server_guard);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_source_cached_only() {
|
||||
let http_server_guard = crate::test_util::http_server();
|
||||
let (_temp_dir, fetcher) = test_setup();
|
||||
let fetcher_1 = fetcher.clone();
|
||||
|
@ -1266,21 +1317,21 @@ mod tests {
|
|||
Url::parse("http://localhost:4545/tests/002_hello.ts").unwrap();
|
||||
let module_url_1 = module_url.clone();
|
||||
let module_url_2 = module_url.clone();
|
||||
// file hasn't been cached before and remote downloads are not allowed
|
||||
// file hasn't been cached before
|
||||
let fut = fetcher
|
||||
.get_source_file_async(&module_url, true, true)
|
||||
.get_source_file_async(&module_url, true, false, true)
|
||||
.then(move |result| {
|
||||
assert!(result.is_err());
|
||||
let err = result.err().unwrap();
|
||||
assert_eq!(err.kind(), ErrorKind::NotFound);
|
||||
assert_eq!(err.kind(), ErrorKind::PermissionDenied);
|
||||
|
||||
// download and cache file
|
||||
fetcher_1.get_source_file_async(&module_url_1, true, false)
|
||||
fetcher_1.get_source_file_async(&module_url_1, true, false, false)
|
||||
})
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
// module is already cached, should be ok even with `no_remote_fetch`
|
||||
fetcher_2.get_source_file_async(&module_url_2, true, true)
|
||||
// module is already cached, should be ok even with `cached_only`
|
||||
fetcher_2.get_source_file_async(&module_url_2, true, false, true)
|
||||
})
|
||||
.then(move |result| {
|
||||
assert!(result.is_ok());
|
||||
|
|
49
cli/flags.rs
49
cli/flags.rs
|
@ -84,7 +84,8 @@ pub struct DenoFlags {
|
|||
pub allow_run: bool,
|
||||
pub allow_hrtime: bool,
|
||||
pub no_prompts: bool,
|
||||
pub no_fetch: bool,
|
||||
pub no_remote: bool,
|
||||
pub cached_only: bool,
|
||||
pub seed: Option<u64>,
|
||||
pub v8_flags: Option<Vec<String>>,
|
||||
// Use tokio::runtime::current_thread
|
||||
|
@ -400,6 +401,7 @@ fn fetch_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
|
|||
lock_args_parse(flags, matches);
|
||||
importmap_arg_parse(flags, matches);
|
||||
config_arg_parse(flags, matches);
|
||||
no_remote_arg_parse(flags, matches);
|
||||
if let Some(file) = matches.value_of("file") {
|
||||
flags.argv.push(file.into());
|
||||
}
|
||||
|
@ -422,6 +424,7 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
|
|||
importmap_arg_parse(flags, matches);
|
||||
config_arg_parse(flags, matches);
|
||||
v8_flags_arg_parse(flags, matches);
|
||||
no_remote_arg_parse(flags, matches);
|
||||
|
||||
if matches.is_present("allow-read") {
|
||||
if matches.value_of("allow-read").is_some() {
|
||||
|
@ -474,8 +477,8 @@ fn run_test_args_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
|
|||
flags.allow_write = true;
|
||||
flags.allow_hrtime = true;
|
||||
}
|
||||
if matches.is_present("no-fetch") {
|
||||
flags.no_fetch = true;
|
||||
if matches.is_present("cached-only") {
|
||||
flags.cached_only = true;
|
||||
}
|
||||
|
||||
if matches.is_present("current-thread") {
|
||||
|
@ -873,6 +876,7 @@ fn fetch_subcommand<'a, 'b>() -> App<'a, 'b> {
|
|||
.arg(lock_write_arg())
|
||||
.arg(importmap_arg())
|
||||
.arg(config_arg())
|
||||
.arg(no_remote_arg())
|
||||
.arg(Arg::with_name("file").takes_value(true).required(true))
|
||||
.about("Fetch the dependencies")
|
||||
.long_about(
|
||||
|
@ -899,6 +903,7 @@ fn run_test_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
|
|||
.arg(config_arg())
|
||||
.arg(lock_arg())
|
||||
.arg(lock_write_arg())
|
||||
.arg(no_remote_arg())
|
||||
.arg(v8_flags_arg())
|
||||
.arg(
|
||||
Arg::with_name("allow-read")
|
||||
|
@ -949,9 +954,9 @@ fn run_test_args<'a, 'b>(app: App<'a, 'b>) -> App<'a, 'b> {
|
|||
.help("Allow all permissions"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("no-fetch")
|
||||
.long("no-fetch")
|
||||
.help("Do not download remote modules"),
|
||||
Arg::with_name("cached-only")
|
||||
.long("cached-only")
|
||||
.help("Require that remote dependencies are already cached"),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("current-thread")
|
||||
|
@ -1150,6 +1155,18 @@ fn v8_flags_arg_parse(flags: &mut DenoFlags, matches: &ArgMatches) {
|
|||
}
|
||||
}
|
||||
|
||||
fn no_remote_arg<'a, 'b>() -> Arg<'a, 'b> {
|
||||
Arg::with_name("no-remote")
|
||||
.long("no-remote")
|
||||
.help("Do not resolve remote modules")
|
||||
}
|
||||
|
||||
fn no_remote_arg_parse(flags: &mut DenoFlags, matches: &clap::ArgMatches) {
|
||||
if matches.is_present("no-remote") {
|
||||
flags.no_remote = true;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO(ry) move this to utility module and add test.
|
||||
/// Strips fragment part of URL. Panics on bad URL.
|
||||
pub fn resolve_urls(urls: Vec<String>) -> Vec<String> {
|
||||
|
@ -2063,14 +2080,28 @@ mod tests {
|
|||
*/
|
||||
|
||||
#[test]
|
||||
fn no_fetch() {
|
||||
let r = flags_from_vec_safe(svec!["deno", "--no-fetch", "script.ts"]);
|
||||
fn no_remote() {
|
||||
let r = flags_from_vec_safe(svec!["deno", "--no-remote", "script.ts"]);
|
||||
assert_eq!(
|
||||
r.unwrap(),
|
||||
DenoFlags {
|
||||
subcommand: DenoSubcommand::Run,
|
||||
argv: svec!["deno", "script.ts"],
|
||||
no_fetch: true,
|
||||
no_remote: true,
|
||||
..DenoFlags::default()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_only() {
|
||||
let r = flags_from_vec_safe(svec!["deno", "--cached-only", "script.ts"]);
|
||||
assert_eq!(
|
||||
r.unwrap(),
|
||||
DenoFlags {
|
||||
subcommand: DenoSubcommand::Run,
|
||||
argv: svec!["deno", "script.ts"],
|
||||
cached_only: true,
|
||||
..DenoFlags::default()
|
||||
}
|
||||
);
|
||||
|
|
|
@ -75,7 +75,8 @@ impl ThreadSafeGlobalState {
|
|||
progress.clone(),
|
||||
!flags.reload,
|
||||
flags.cache_blacklist.clone(),
|
||||
flags.no_fetch,
|
||||
flags.no_remote,
|
||||
flags.cached_only,
|
||||
)?;
|
||||
|
||||
let ts_compiler = TsCompiler::new(
|
||||
|
|
1
cli/tests/035_cached_only_flag.out
Normal file
1
cli/tests/035_cached_only_flag.out
Normal file
|
@ -0,0 +1 @@
|
|||
Cannot find module "http://127.0.0.1:4545/cli/tests/019_media_types.ts" in cache, --cached-only is specified
|
|
@ -296,10 +296,10 @@ itest!(_034_onload {
|
|||
output: "034_onload.out",
|
||||
});
|
||||
|
||||
itest!(_035_no_fetch_flag {
|
||||
itest!(_035_cached_only_flag {
|
||||
args:
|
||||
"--reload --no-fetch http://127.0.0.1:4545/cli/tests/019_media_types.ts",
|
||||
output: "035_no_fetch_flag.out",
|
||||
"--reload --cached-only http://127.0.0.1:4545/cli/tests/019_media_types.ts",
|
||||
output: "035_cached_only_flag.out",
|
||||
exit_code: 1,
|
||||
check_stderr: true,
|
||||
http_server: true,
|
||||
|
@ -397,6 +397,15 @@ itest!(_051_wasm_import {
|
|||
http_server: true,
|
||||
});
|
||||
|
||||
itest!(_052_no_remote_flag {
|
||||
args:
|
||||
"--reload --no-remote http://127.0.0.1:4545/cli/tests/019_media_types.ts",
|
||||
output: "052_no_remote_flag.out",
|
||||
exit_code: 1,
|
||||
check_stderr: true,
|
||||
http_server: true,
|
||||
});
|
||||
|
||||
itest!(lock_check_ok {
|
||||
args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
|
||||
output: "003_relative_import.ts.out",
|
||||
|
|
Loading…
Reference in a new issue