feat(unstable): Respect --cached-only flags for npm: specifiers (#15512)

This commit changes "npm:" specifier handling to respect "--cached-only" flags and adds "Download" messages for npm registry api calls.

Co-authored-by: David Sherret <dsherret@gmail.com>

cli/npm/cache.rs

@@ -7,12 +7,14 @@ use std::path::PathBuf;
 use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
+use deno_core::error::custom_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
 use deno_runtime::colors;
 use deno_runtime::deno_fetch::reqwest;
 
 use crate::deno_dir::DenoDir;
+use crate::file_fetcher::CacheSetting;
 use crate::fs_util;
 
 use super::tarball::verify_and_extract_tarball;
@@ -152,15 +154,24 @@ impl ReadonlyNpmCache {
 
 /// Stores a single copy of npm packages in a cache.
 #[derive(Clone, Debug)]
-pub struct NpmCache(ReadonlyNpmCache);
+pub struct NpmCache {
+  readonly: ReadonlyNpmCache,
+  cache_setting: CacheSetting,
+}
 
 impl NpmCache {
-  pub fn from_deno_dir(dir: &DenoDir) -> Result<Self, AnyError> {
+  pub fn from_deno_dir(
-    Ok(Self(ReadonlyNpmCache::from_deno_dir(dir)?))
+    dir: &DenoDir,
+    cache_setting: CacheSetting,
+  ) -> Result<Self, AnyError> {
+    Ok(Self {
+      readonly: ReadonlyNpmCache::from_deno_dir(dir)?,
+      cache_setting,
+    })
   }
 
   pub fn as_readonly(&self) -> ReadonlyNpmCache {
-    self.0.clone()
+    self.readonly.clone()
   }
 
   pub async fn ensure_package(
@@ -169,13 +180,22 @@ impl NpmCache {
     dist: &NpmPackageVersionDistInfo,
     registry_url: &Url,
   ) -> Result<(), AnyError> {
-    let package_folder = self.0.package_folder(id, registry_url);
+    let package_folder = self.readonly.package_folder(id, registry_url);
     if package_folder.exists()
       // if this file exists, then the package didn't successfully extract
       // the first time, or another process is currently extracting the zip file
       && !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists()
     {
       return Ok(());
+    } else if self.cache_setting == CacheSetting::Only {
+      return Err(custom_error(
+        "NotCached",
+        format!(
+          "An npm specifier not found in cache: \"{}\", --cached-only is specified.",
+          id.name
+        )
+      )
+      );
     }
 
     log::log!(
@@ -225,15 +245,15 @@ impl NpmCache {
     id: &NpmPackageId,
     registry_url: &Url,
   ) -> PathBuf {
-    self.0.package_folder(id, registry_url)
+    self.readonly.package_folder(id, registry_url)
   }
 
   pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf {
-    self.0.package_name_folder(name, registry_url)
+    self.readonly.package_name_folder(name, registry_url)
   }
 
   pub fn registry_folder(&self, registry_url: &Url) -> PathBuf {
-    self.0.registry_folder(registry_url)
+    self.readonly.registry_folder(registry_url)
   }
 
   pub fn resolve_package_id_from_specifier(
@@ -242,7 +262,7 @@ impl NpmCache {
     registry_url: &Url,
   ) -> Result<NpmPackageId, AnyError> {
     self
-      .0
+      .readonly
       .resolve_package_id_from_specifier(specifier, registry_url)
   }
 }

cli/npm/mod.rs

@@ -29,6 +29,7 @@ use registry::NpmRegistryApi;
 use resolution::NpmResolution;
 
 use crate::deno_dir::DenoDir;
+use crate::file_fetcher::CacheSetting;
 
 use self::cache::ReadonlyNpmCache;
 use self::resolution::NpmResolutionSnapshot;
@@ -77,12 +78,24 @@ pub struct GlobalNpmPackageResolver {
 }
 
 impl GlobalNpmPackageResolver {
-  pub fn from_deno_dir(dir: &DenoDir, reload: bool) -> Result<Self, AnyError> {
+  pub fn from_deno_dir(
-    Ok(Self::from_cache(NpmCache::from_deno_dir(dir)?, reload))
+    dir: &DenoDir,
+    reload: bool,
+    cache_setting: CacheSetting,
+  ) -> Result<Self, AnyError> {
+    Ok(Self::from_cache(
+      NpmCache::from_deno_dir(dir, cache_setting.clone())?,
+      reload,
+      cache_setting,
+    ))
   }
 
-  fn from_cache(cache: NpmCache, reload: bool) -> Self {
+  fn from_cache(
-    let api = NpmRegistryApi::new(cache.clone(), reload);
+    cache: NpmCache,
+    reload: bool,
+    cache_setting: CacheSetting,
+  ) -> Self {
+    let api = NpmRegistryApi::new(cache.clone(), reload, cache_setting);
     let registry_url = api.base_url().to_owned();
     let resolution = Arc::new(NpmResolution::new(api));
 

cli/npm/registry.rs

@@ -2,11 +2,13 @@
 
 use std::collections::HashMap;
 use std::fs;
+use std::io::ErrorKind;
 use std::path::PathBuf;
 use std::sync::Arc;
 
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
+use deno_core::error::custom_error;
 use deno_core::error::AnyError;
 use deno_core::parking_lot::Mutex;
 use deno_core::serde::Deserialize;
@@ -16,6 +18,7 @@ use deno_runtime::colors;
 use deno_runtime::deno_fetch::reqwest;
 use serde::Serialize;
 
+use crate::file_fetcher::CacheSetting;
 use crate::fs_util;
 use crate::http_cache::CACHE_PERM;
 
@@ -100,6 +103,7 @@ pub struct NpmRegistryApi {
   cache: NpmCache,
   mem_cache: Arc<Mutex<HashMap<String, Option<NpmPackageInfo>>>>,
   reload: bool,
+  cache_setting: CacheSetting,
 }
 
 impl NpmRegistryApi {
@@ -122,16 +126,26 @@ impl NpmRegistryApi {
     }
   }
 
-  pub fn new(cache: NpmCache, reload: bool) -> Self {
+  pub fn new(
-    Self::from_base(Self::default_url(), cache, reload)
+    cache: NpmCache,
+    reload: bool,
+    cache_setting: CacheSetting,
+  ) -> Self {
+    Self::from_base(Self::default_url(), cache, reload, cache_setting)
   }
 
-  pub fn from_base(base_url: Url, cache: NpmCache, reload: bool) -> Self {
+  pub fn from_base(
+    base_url: Url,
+    cache: NpmCache,
+    reload: bool,
+    cache_setting: CacheSetting,
+  ) -> Self {
     Self {
       base_url,
       cache,
       mem_cache: Default::default(),
       reload,
+      cache_setting,
     }
   }
 
@@ -163,6 +177,7 @@ impl NpmRegistryApi {
         // attempt to load from the file cache
         maybe_package_info = self.load_file_cached_package_info(name);
       }
+
       if maybe_package_info.is_none() {
         maybe_package_info = self
           .load_package_info_from_registry(name)
@@ -191,13 +206,14 @@ impl NpmRegistryApi {
     &self,
     name: &str,
   ) -> Option<NpmPackageInfo> {
-    let file_cache_path = self.get_package_file_cache_path(name);
+    match self.load_file_cached_package_info_result(name) {
-    let file_text = fs::read_to_string(file_cache_path).ok()?;
+      Ok(value) => value,
-    match serde_json::from_str(&file_text) {
-      Ok(result) => Some(result),
       Err(err) => {
         if cfg!(debug_assertions) {
-          panic!("could not deserialize: {:#}", err);
+          panic!(
+            "error loading cached npm package info for {}: {:#}",
+            name, err
+          );
         } else {
           None
         }
@@ -205,22 +221,73 @@ impl NpmRegistryApi {
     }
   }
 
+  fn load_file_cached_package_info_result(
+    &self,
+    name: &str,
+  ) -> Result<Option<NpmPackageInfo>, AnyError> {
+    let file_cache_path = self.get_package_file_cache_path(name);
+    let file_text = match fs::read_to_string(file_cache_path) {
+      Ok(file_text) => file_text,
+      Err(err) if err.kind() == ErrorKind::NotFound => return Ok(None),
+      Err(err) => return Err(err.into()),
+    };
+    Ok(serde_json::from_str(&file_text)?)
+  }
+
   fn save_package_info_to_file_cache(
     &self,
     name: &str,
     package_info: &NpmPackageInfo,
   ) {
+    if let Err(err) =
+      self.save_package_info_to_file_cache_result(name, package_info)
+    {
+      if cfg!(debug_assertions) {
+        panic!(
+          "error saving cached npm package info for {}: {:#}",
+          name, err
+        );
+      }
+    }
+  }
+
+  fn save_package_info_to_file_cache_result(
+    &self,
+    name: &str,
+    package_info: &NpmPackageInfo,
+  ) -> Result<(), AnyError> {
     let file_cache_path = self.get_package_file_cache_path(name);
-    let file_text = serde_json::to_string_pretty(&package_info).unwrap();
+    let file_text = serde_json::to_string(&package_info)?;
-    let _ignore =
+    std::fs::create_dir_all(&file_cache_path.parent().unwrap())?;
-      fs_util::atomic_write_file(&file_cache_path, file_text, CACHE_PERM);
+    fs_util::atomic_write_file(&file_cache_path, file_text, CACHE_PERM)?;
+    Ok(())
   }
 
   async fn load_package_info_from_registry(
     &self,
     name: &str,
   ) -> Result<Option<NpmPackageInfo>, AnyError> {
-    let response = match reqwest::get(self.get_package_url(name)).await {
+    if self.cache_setting == CacheSetting::Only {
+      return Err(custom_error(
+        "NotCached",
+        format!(
+          "An npm specifier not found in cache: \"{}\", --cached-only is specified.",
+          name
+        )
+      )
+      );
+    }
+
+    let package_url = self.get_package_url(name);
+
+    log::log!(
+      log::Level::Info,
+      "{} {}",
+      colors::green("Download"),
+      package_url,
+    );
+
+    let response = match reqwest::get(package_url).await {
       Ok(response) => response,
       Err(err) => {
         // attempt to use the local cache

cli/proc_state.rs

@@ -217,8 +217,11 @@ impl ProcState {
       warn!("{}", ignored_options);
     }
     let emit_cache = EmitCache::new(dir.gen_cache.clone());
-    let npm_resolver =
+    let npm_resolver = GlobalNpmPackageResolver::from_deno_dir(
-      GlobalNpmPackageResolver::from_deno_dir(&dir, cli_options.reload_flag())?;
+      &dir,
+      cli_options.reload_flag(),
+      cli_options.cache_setting(),
+    )?;
 
     Ok(ProcState(Arc::new(Inner {
       dir,

cli/tests/integration/npm_tests.rs

@@ -1,7 +1,10 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
 use deno_core::url::Url;
+use std::process::Stdio;
 use test_util as util;
+use util::assert_contains;
+use util::http_server;
 
 // NOTE: It's possible to automatically update the npm registry data in the test server
 // by setting the DENO_TEST_UTIL_UPDATE_NPM=1 environment variable.
@@ -51,6 +54,13 @@ itest!(dynamic_import {
   http_server: true,
 });
 
+itest!(cached_only {
+  args: "run --cached-only --unstable npm/cached_only/main.ts",
+  output: "npm/cached_only/main.out",
+  envs: env_vars(),
+  exit_code: 1,
+});
+
 itest!(import_map {
   args: "run --allow-read --unstable --import-map npm/import_map/import_map.json npm/import_map/main.js",
   output: "npm/import_map/main.out",
@@ -76,6 +86,83 @@ fn parallel_downloading() {
   assert!(out.contains("chalk cjs loads"));
 }
 
+#[test]
+fn cached_only_after_first_run() {
+  let _server = http_server();
+
+  let deno_dir = util::new_deno_dir();
+
+  let deno = util::deno_cmd_with_deno_dir(&deno_dir)
+    .current_dir(util::testdata_path())
+    .arg("run")
+    .arg("--unstable")
+    .arg("--allow-read")
+    .arg("--allow-env")
+    .arg("npm/cached_only_after_first_run/main1.ts")
+    .env("NO_COLOR", "1")
+    .envs(env_vars())
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn()
+    .unwrap();
+  let output = deno.wait_with_output().unwrap();
+  let stderr = String::from_utf8_lossy(&output.stderr);
+  let stdout = String::from_utf8_lossy(&output.stdout);
+  assert_contains!(stderr, "Download");
+  assert_contains!(stdout, "createChalk: chalk");
+  assert!(output.status.success());
+
+  let deno = util::deno_cmd_with_deno_dir(&deno_dir)
+    .current_dir(util::testdata_path())
+    .arg("run")
+    .arg("--unstable")
+    .arg("--allow-read")
+    .arg("--allow-env")
+    .arg("--cached-only")
+    .arg("npm/cached_only_after_first_run/main2.ts")
+    .env("NO_COLOR", "1")
+    .envs(env_vars())
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn()
+    .unwrap();
+  let output = deno.wait_with_output().unwrap();
+  let stderr = String::from_utf8_lossy(&output.stderr);
+  let stdout = String::from_utf8_lossy(&output.stdout);
+  assert_contains!(
+    stderr,
+    "An npm specifier not found in cache: \"ansi-styles\", --cached-only is specified."
+  );
+  assert!(stdout.is_empty());
+  assert!(!output.status.success());
+
+  let deno = util::deno_cmd_with_deno_dir(&deno_dir)
+    .current_dir(util::testdata_path())
+    .arg("run")
+    .arg("--unstable")
+    .arg("--allow-read")
+    .arg("--allow-env")
+    .arg("--cached-only")
+    .arg("npm/cached_only_after_first_run/main1.ts")
+    .env("NO_COLOR", "1")
+    .envs(env_vars())
+    .stdout(Stdio::piped())
+    .stderr(Stdio::piped())
+    .spawn()
+    .unwrap();
+
+  eprintln!("DENO DIR: {}", deno_dir.path().display());
+  std::mem::forget(deno_dir);
+  let output = deno.wait_with_output().unwrap();
+  let stderr = String::from_utf8_lossy(&output.stderr);
+  let stdout = String::from_utf8_lossy(&output.stdout);
+  eprintln!("stderr {}", stderr);
+  eprintln!("stdout {}", stdout);
+  assert!(output.status.success());
+  assert!(stderr.is_empty());
+  assert_contains!(stdout, "createChalk: chalk");
+}
+
 #[test]
 fn ensure_registry_files_local() {
   // ensures the registry files all point at local tarballs

cli/tests/testdata/npm/cached_only/main.out

@@ -0,0 +1,4 @@
+error: Error getting response at http://localhost:4545/npm/registry/chalk
+
+Caused by:
+    An npm specifier not found in cache: "chalk", --cached-only is specified.

cli/tests/testdata/npm/cached_only/main.ts

@@ -0,0 +1,3 @@
+import chalk from "npm:chalk@5";
+
+console.log(chalk);

cli/tests/testdata/npm/cached_only_after_first_run/main1.ts

@@ -0,0 +1,3 @@
+import chalk from "npm:chalk@5";
+
+console.log(chalk);

cli/tests/testdata/npm/cached_only_after_first_run/main2.ts

@@ -0,0 +1,3 @@
+import chalk from "npm:chalk@4";
+
+console.log(chalk);

cli/tests/testdata/npm/cjs_sub_path/main.out

@@ -1,3 +1,18 @@
+Download http://localhost:4545/npm/registry/ajv
+Download http://localhost:4545/npm/registry/ajv-formats
+Download http://localhost:4545/npm/registry/chai
+Download http://localhost:4545/npm/registry/fast-deep-equal
+Download http://localhost:4545/npm/registry/json-schema-traverse
+Download http://localhost:4545/npm/registry/require-from-string
+Download http://localhost:4545/npm/registry/uri-js
+Download http://localhost:4545/npm/registry/assertion-error
+Download http://localhost:4545/npm/registry/check-error
+Download http://localhost:4545/npm/registry/deep-eql
+Download http://localhost:4545/npm/registry/get-func-name
+Download http://localhost:4545/npm/registry/loupe
+Download http://localhost:4545/npm/registry/pathval
+Download http://localhost:4545/npm/registry/type-detect
+Download http://localhost:4545/npm/registry/punycode
 Download http://localhost:4545/npm/registry/ajv/ajv-8.11.0.tgz
 Download http://localhost:4545/npm/registry/ajv-formats/ajv-formats-2.1.1.tgz
 Download http://localhost:4545/npm/registry/assertion-error/assertion-error-1.1.0.tgz

cli/tests/testdata/npm/cjs_with_deps/main.out

@@ -1,3 +1,17 @@
+Download http://localhost:4545/npm/registry/chai
+Download http://localhost:4545/npm/registry/chalk
+Download http://localhost:4545/npm/registry/assertion-error
+Download http://localhost:4545/npm/registry/check-error
+Download http://localhost:4545/npm/registry/deep-eql
+Download http://localhost:4545/npm/registry/get-func-name
+Download http://localhost:4545/npm/registry/loupe
+Download http://localhost:4545/npm/registry/pathval
+Download http://localhost:4545/npm/registry/type-detect
+Download http://localhost:4545/npm/registry/ansi-styles
+Download http://localhost:4545/npm/registry/supports-color
+Download http://localhost:4545/npm/registry/color-convert
+Download http://localhost:4545/npm/registry/has-flag
+Download http://localhost:4545/npm/registry/color-name
 Download http://localhost:4545/npm/registry/ansi-styles/ansi-styles-4.3.0.tgz
 Download http://localhost:4545/npm/registry/assertion-error/assertion-error-1.1.0.tgz
 Download http://localhost:4545/npm/registry/chai/chai-4.3.6.tgz

cli/tests/testdata/npm/dynamic_import/main.out

@@ -1,4 +1,5 @@
 A
+Download http://localhost:4545/npm/registry/chalk
 Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
 B
 C

cli/tests/testdata/npm/esm/main.out

@@ -1,2 +1,3 @@
+Download http://localhost:4545/npm/registry/chalk
 Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
 chalk esm loads

cli/tests/testdata/npm/esm/test.out

@@ -1,3 +1,4 @@
+Download http://localhost:4545/npm/registry/chalk
 Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
 Check [WILDCARD]/std/node/module_all.ts
 chalk esm loads

cli/tests/testdata/npm/import_map/main.out

@@ -1,2 +1,3 @@
+Download http://localhost:4545/npm/registry/chalk
 Download http://localhost:4545/npm/registry/chalk/chalk-5.0.1.tgz
 chalk import map loads