refactor(cli): move snapshot_from_lockfile function to deno_npm (#20024)

This commit moves `snapshot_from_lockfile` function to [deno_npm
crate](https://github.com/denoland/deno_npm). This allows this function
to be called outside Deno CLI (in particular, Deno Deploy).

Cargo.lock

@@ -1248,9 +1248,9 @@ dependencies = [
 
 [[package]]
 name = "deno_lockfile"
-version = "0.14.1"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa35b176a415501662244f99846b3d0dd1a7792b2135bf7f80007f8eca8b24ac"
+checksum = "3e1fcc91fa4e18c3e0574965d7133709e76eda665cb589de703219f0819dfaec"
 dependencies = [
  "ring",
  "serde",
@@ -1353,12 +1353,13 @@ dependencies = [
 
 [[package]]
 name = "deno_npm"
-version = "0.10.1"
+version = "0.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa5d1097de53e8ce3316d3e44095e253719ae367cf7478263f83082f44dddabf"
+checksum = "341f2c3935bee51c15203c587213c42d120f0dc56f0aca912a4bebbf038d1030"
 dependencies = [
  "anyhow",
  "async-trait",
+ "deno_lockfile",
  "deno_semver",
  "futures",
  "log",
@@ -1965,9 +1966,9 @@ dependencies = [
 
 [[package]]
 name = "eszip"
-version = "0.48.0"
+version = "0.49.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58d2382a70396ccffd260adb879ed9f954c45457e8075830c7d4ec30c6dc8ad2"
+checksum = "0b32387a359104b88abe2583679e6dcb817aff5534204ade9e4c4528db8079f3"
 dependencies = [
  "anyhow",
  "base64 0.21.0",

Cargo.toml

@@ -47,9 +47,9 @@ deno_runtime = { version = "0.122.0", path = "./runtime" }
 napi_sym = { version = "0.44.0", path = "./cli/napi/sym" }
 deno_bench_util = { version = "0.108.0", path = "./bench_util" }
 test_util = { path = "./test_util" }
-deno_lockfile = "0.14.1"
+deno_lockfile = "0.15.0"
 deno_media_type = { version = "0.1.1", features = ["module_specifier"] }
-deno_npm = "0.10.1"
+deno_npm = "0.11.0"
 deno_semver = "0.3.0"
 
 # exts

cli/Cargo.toml

@@ -57,7 +57,7 @@ deno_npm.workspace = true
 deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_semver.workspace = true
 deno_task_shell = "=0.13.1"
-eszip = "=0.48.0"
+eszip = "=0.49.0"
 napi_sym.workspace = true
 
 async-trait.workspace = true

cli/args/lockfile.rs

@@ -1,25 +1,14 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use std::collections::HashMap;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-use deno_core::anyhow::bail;
-use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
-use deno_core::futures::stream::FuturesOrdered;
-use deno_core::futures::StreamExt;
 use deno_core::parking_lot::Mutex;
 use deno_npm::registry::NpmRegistryApi;
-use deno_npm::resolution::SerializedNpmResolutionSnapshot;
-use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;
 use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
-use deno_npm::NpmPackageId;
-use deno_npm::NpmResolutionPackageSystemInfo;
-use deno_semver::npm::NpmPackageReq;
 
 use crate::args::ConfigFile;
-use crate::npm::CliNpmRegistryApi;
 use crate::Flags;
 
 use super::DenoSubcommand;
@@ -63,96 +52,14 @@ pub fn discover(
 
 pub async fn snapshot_from_lockfile(
   lockfile: Arc<Mutex<Lockfile>>,
-  api: &CliNpmRegistryApi,
+  api: &dyn NpmRegistryApi,
 ) -> Result<ValidSerializedNpmResolutionSnapshot, AnyError> {
-  let (root_packages, mut packages) = {
-    let lockfile = lockfile.lock();
-
-    let mut root_packages =
-      HashMap::<NpmPackageReq, NpmPackageId>::with_capacity(
-        lockfile.content.npm.specifiers.len(),
-      );
-    // collect the specifiers to version mappings
-    for (key, value) in &lockfile.content.npm.specifiers {
-      let package_req = NpmPackageReq::from_str(key)
-        .with_context(|| format!("Unable to parse npm specifier: {key}"))?;
-      let package_id = NpmPackageId::from_serialized(value)?;
-      root_packages.insert(package_req, package_id.clone());
-    }
-
-    // now fill the packages except for the dist information
-    let mut packages = Vec::with_capacity(lockfile.content.npm.packages.len());
-    for (key, package) in &lockfile.content.npm.packages {
-      let id = NpmPackageId::from_serialized(key)?;
-
-      // collect the dependencies
-      let mut dependencies = HashMap::with_capacity(package.dependencies.len());
-      for (name, specifier) in &package.dependencies {
-        let dep_id = NpmPackageId::from_serialized(specifier)?;
-        dependencies.insert(name.clone(), dep_id);
-      }
-
-      packages.push(SerializedNpmResolutionSnapshotPackage {
-        id,
-        dependencies,
-        // temporarily empty
-        system: Default::default(),
-        dist: Default::default(),
-        optional_dependencies: Default::default(),
-      });
-    }
-    (root_packages, packages)
+  let incomplete_snapshot = {
+    let lock = lockfile.lock();
+    deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)?
   };
-
-  // now that the lockfile is dropped, fetch the package version information
-  let pkg_nvs = packages.iter().map(|p| p.id.nv.clone()).collect::<Vec<_>>();
-  let get_version_infos = || {
-    FuturesOrdered::from_iter(pkg_nvs.iter().map(|nv| async move {
-      let package_info = api.package_info(&nv.name).await?;
-      match package_info.version_info(nv) {
-        Ok(version_info) => Ok(version_info),
-        Err(err) => {
-          bail!("Could not find '{}' specified in the lockfile.", err.0);
-        }
-      }
-    }))
-  };
-  let mut version_infos = get_version_infos();
-  let mut i = 0;
-  while let Some(result) = version_infos.next().await {
-    match result {
-      Ok(version_info) => {
-        let package = &mut packages[i];
-        package.dist = version_info.dist;
-        package.system = NpmResolutionPackageSystemInfo {
-          cpu: version_info.cpu,
-          os: version_info.os,
-        };
-        package.optional_dependencies =
-          version_info.optional_dependencies.into_keys().collect();
-      }
-      Err(err) => {
-        if api.mark_force_reload() {
-          // reset and try again
-          version_infos = get_version_infos();
-          i = 0;
-          continue;
-        } else {
-          return Err(err);
-        }
-      }
-    }
-
-    i += 1;
-  }
-
-  // clear the memory cache to reduce memory usage
-  api.clear_memory_cache();
-
-  SerializedNpmResolutionSnapshot {
-    packages,
-    root_packages,
-  }
-  .into_valid()
-  .context("The lockfile is corrupt. You can recreate it with --lock-write")
+  let snapshot =
+    deno_npm::resolution::snapshot_from_lockfile(incomplete_snapshot, api)
+      .await?;
+  Ok(snapshot)
 }

cli/args/mod.rs

@@ -818,16 +818,17 @@ impl CliOptions {
 
     if let Some(lockfile) = self.maybe_lockfile() {
       if !lockfile.lock().overwrite {
-        return Ok(Some(
-          snapshot_from_lockfile(lockfile.clone(), api)
+        let snapshot = snapshot_from_lockfile(lockfile.clone(), api)
           .await
           .with_context(|| {
             format!(
               "failed reading lockfile '{}'",
               lockfile.lock().filename.display()
             )
-            })?,
-        ));
+          })?;
+        // clear the memory cache to reduce memory usage
+        api.clear_memory_cache();
+        return Ok(Some(snapshot));
       }
     }
 

cli/lsp/language_server.rs

@@ -927,6 +927,25 @@ impl Inner {
     Ok(())
   }
 
+  async fn get_npm_snapshot(
+    &self,
+  ) -> Option<ValidSerializedNpmResolutionSnapshot> {
+    let lockfile = self.config.maybe_lockfile()?;
+    let snapshot =
+      match snapshot_from_lockfile(lockfile.clone(), &*self.npm.api).await {
+        Ok(snapshot) => snapshot,
+        Err(err) => {
+          lsp_warn!("Failed getting npm snapshot from lockfile: {}", err);
+          return None;
+        }
+      };
+
+    // clear the memory cache to reduce memory usage
+    self.npm.api.clear_memory_cache();
+
+    Some(snapshot)
+  }
+
   async fn recreate_npm_services_if_necessary(&mut self) {
     let deno_dir = match DenoDir::new(self.maybe_global_cache_path.clone()) {
       Ok(deno_dir) => deno_dir,
@@ -948,18 +967,7 @@ impl Inner {
       registry_url,
       &progress_bar,
     );
-    let maybe_snapshot = match self.config.maybe_lockfile() {
-      Some(lockfile) => {
-        match snapshot_from_lockfile(lockfile.clone(), &self.npm.api).await {
-          Ok(snapshot) => Some(snapshot),
-          Err(err) => {
-            lsp_warn!("Failed getting npm snapshot from lockfile: {}", err);
-            None
-          }
-        }
-      }
-      None => None,
-    };
+    let maybe_snapshot = self.get_npm_snapshot().await;
     (self.npm.resolver, self.npm.resolution) =
       create_npm_resolver_and_resolution(
         registry_url,

cli/npm/registry.rs

@@ -99,27 +99,6 @@ impl CliNpmRegistryApi {
     &self.inner().base_url
   }
 
-  /// Marks that new requests for package information should retrieve it
-  /// from the npm registry
-  ///
-  /// Returns true if it was successfully set for the first time.
-  pub fn mark_force_reload(&self) -> bool {
-    // never force reload the registry information if reloading
-    // is disabled or if we're already reloading
-    if matches!(
-      self.inner().cache.cache_setting(),
-      CacheSetting::Only | CacheSetting::ReloadAll
-    ) {
-      return false;
-    }
-    if self.inner().force_reload_flag.raise() {
-      self.clear_memory_cache(); // clear the memory cache to force reloading
-      true
-    } else {
-      false
-    }
-  }
-
   fn inner(&self) -> &Arc<CliNpmRegistryApiInner> {
     // this panicking indicates a bug in the code where this
     // wasn't initialized
@@ -154,6 +133,23 @@ impl NpmRegistryApi for CliNpmRegistryApi {
       }
     }
   }
+
+  fn mark_force_reload(&self) -> bool {
+    // never force reload the registry information if reloading
+    // is disabled or if we're already reloading
+    if matches!(
+      self.inner().cache.cache_setting(),
+      CacheSetting::Only | CacheSetting::ReloadAll
+    ) {
+      return false;
+    }
+    if self.inner().force_reload_flag.raise() {
+      self.clear_memory_cache(); // clear the cache to force reloading
+      true
+    } else {
+      false
+    }
+  }
 }
 
 type CacheItemPendingResult =
@@ -386,7 +382,7 @@ impl CliNpmRegistryApiInner {
     name_folder_path.join("registry.json")
   }
 
-  pub fn clear_memory_cache(&self) {
+  fn clear_memory_cache(&self) {
     self.mem_cache.lock().clear();
   }
 

cli/npm/resolution.rs

@@ -11,6 +11,7 @@ use deno_core::TaskQueue;
 use deno_lockfile::NpmPackageDependencyLockfileInfo;
 use deno_lockfile::NpmPackageLockfileInfo;
 use deno_npm::registry::NpmPackageInfo;
+use deno_npm::registry::NpmRegistryApi;
 use deno_npm::resolution::NpmPackageVersionResolutionError;
 use deno_npm::resolution::NpmPackagesPartitioned;
 use deno_npm::resolution::NpmResolutionError;

cli/tests/integration/npm_tests.rs

@@ -1915,7 +1915,8 @@ fn reload_info_not_found_cache_but_exists_remote() {
       "error: failed reading lockfile '[WILDCARD]deno.lock'\n",
       "\n",
       "Caused by:\n",
-      "    Could not find '@denotest/esm-basic@1.0.0' specified in the lockfile.\n"
+      "    0: Could not find '@denotest/esm-basic@1.0.0' specified in the lockfile.\n",
+      "    1: Could not find version '1.0.0' for npm package '@denotest/esm-basic'.\n",
     ));
     output.assert_exit_code(1);