From 148694eb351ea3f733852b7786a3268617811e27 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 2 Oct 2023 17:53:55 -0400 Subject: [PATCH] refactor(npm): make `NpmCache`, `CliNpmRegistryApi`, and `NpmResolution` internal to `npm::managed` (#20764) --- cli/args/lockfile.rs | 18 -- cli/args/mod.rs | 74 ++++-- cli/factory.rs | 142 +++------- cli/lsp/analysis.rs | 12 +- cli/lsp/language_server.rs | 246 ++++++----------- cli/lsp/npm.rs | 4 +- cli/npm/cache_dir.rs | 268 +++++++++++++++++++ cli/npm/{ => managed}/cache.rs | 394 +++++----------------------- cli/npm/managed/installer.rs | 2 +- cli/npm/managed/mod.rs | 285 ++++++++++++++++++-- cli/npm/{ => managed}/registry.rs | 23 -- cli/npm/managed/resolution.rs | 14 +- cli/npm/managed/resolvers/common.rs | 2 +- cli/npm/managed/resolvers/global.rs | 3 +- cli/npm/managed/resolvers/local.rs | 6 +- cli/npm/managed/resolvers/mod.rs | 5 +- cli/npm/{ => managed}/tarball.rs | 0 cli/npm/mod.rs | 53 ++-- cli/standalone/binary.rs | 42 ++- cli/standalone/mod.rs | 129 ++++----- cli/tools/info.rs | 2 +- cli/tools/task.rs | 6 +- cli/tools/vendor/mod.rs | 11 +- 23 files changed, 894 insertions(+), 847 deletions(-) create mode 100644 cli/npm/cache_dir.rs rename cli/npm/{ => managed}/cache.rs (54%) rename cli/npm/{ => managed}/registry.rs (93%) rename cli/npm/{ => managed}/tarball.rs (100%) diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index dd976862e4..5593670786 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -1,12 +1,8 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. use std::path::PathBuf; -use std::sync::Arc; use deno_core::error::AnyError; -use deno_core::parking_lot::Mutex; -use deno_npm::registry::NpmRegistryApi; -use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use crate::args::ConfigFile; use crate::Flags; @@ -49,17 +45,3 @@ pub fn discover( let lockfile = Lockfile::new(filename, flags.lock_write)?; Ok(Some(lockfile)) } - -pub async fn snapshot_from_lockfile( - lockfile: Arc>, - api: &dyn NpmRegistryApi, -) -> Result { - let incomplete_snapshot = { - let lock = lockfile.lock(); - deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)? - }; - let snapshot = - deno_npm::resolution::snapshot_from_lockfile(incomplete_snapshot, api) - .await?; - Ok(snapshot) -} diff --git a/cli/args/mod.rs b/cli/args/mod.rs index e6ada94533..d91affed10 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -7,7 +7,6 @@ mod lockfile; pub mod package_json; pub use self::import_map::resolve_import_map_from_specifier; -pub use self::lockfile::snapshot_from_lockfile; use self::package_json::PackageJsonDeps; use ::import_map::ImportMap; use deno_core::resolve_url_or_path; @@ -55,6 +54,8 @@ use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::PermissionsOptions; use once_cell::sync::Lazy; use once_cell::sync::OnceCell; +use serde::Deserialize; +use serde::Serialize; use std::collections::HashMap; use std::env; use std::io::BufReader; @@ -67,8 +68,6 @@ use std::sync::Arc; use thiserror::Error; use crate::file_fetcher::FileFetcher; -use crate::npm::CliNpmRegistryApi; -use crate::npm::NpmProcessState; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::glob::expand_globs; use crate::version; @@ -77,6 +76,28 @@ use deno_config::FmtConfig; use deno_config::LintConfig; use deno_config::TestConfig; +static NPM_REGISTRY_DEFAULT_URL: Lazy = Lazy::new(|| { + let env_var_name = "NPM_CONFIG_REGISTRY"; + if let Ok(registry_url) = std::env::var(env_var_name) { + // ensure there is a trailing slash for the directory + let registry_url = format!("{}/", registry_url.trim_end_matches('/')); + match Url::parse(®istry_url) { + Ok(url) => { + return url; + } + Err(err) => { + log::debug!("Invalid {} environment variable: {:#}", env_var_name, err,); + } + } + } + + Url::parse("https://registry.npmjs.org").unwrap() +}); + +pub fn npm_registry_default_url() -> &'static Url { + &NPM_REGISTRY_DEFAULT_URL +} + pub fn ts_config_to_emit_options( config: deno_config::TsConfig, ) -> deno_ast::EmitOptions { @@ -545,6 +566,13 @@ pub fn get_root_cert_store( Ok(root_cert_store) } +/// State provided to the process via an environment variable. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NpmProcessState { + pub snapshot: deno_npm::resolution::SerializedNpmResolutionSnapshot, + pub local_node_modules_path: Option, +} + const RESOLUTION_STATE_ENV_VAR_NAME: &str = "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; @@ -560,7 +588,7 @@ static NPM_PROCESS_STATE: Lazy> = Lazy::new(|| { /// Overrides for the options below that when set will /// use these values over the values derived from the /// CLI flags or config file. -#[derive(Default)] +#[derive(Default, Clone)] struct CliOptionOverrides { import_map_specifier: Option>, } @@ -843,32 +871,15 @@ impl CliOptions { } } - pub async fn resolve_npm_resolution_snapshot( + pub fn resolve_npm_resolution_snapshot( &self, - api: &CliNpmRegistryApi, ) -> Result, AnyError> { if let Some(state) = &*NPM_PROCESS_STATE { // TODO(bartlomieju): remove this clone - return Ok(Some(state.snapshot.clone().into_valid()?)); + Ok(Some(state.snapshot.clone().into_valid()?)) + } else { + Ok(None) } - - if let Some(lockfile) = self.maybe_lockfile() { - if !lockfile.lock().overwrite { - let snapshot = snapshot_from_lockfile(lockfile.clone(), api) - .await - .with_context(|| { - format!( - "failed reading lockfile '{}'", - lockfile.lock().filename.display() - ) - })?; - // clear the memory cache to reduce memory usage - api.clear_memory_cache(); - return Ok(Some(snapshot)); - } - } - - Ok(None) } // If the main module should be treated as being in an npm package. @@ -892,6 +903,19 @@ impl CliOptions { self.maybe_node_modules_folder.clone() } + pub fn with_node_modules_dir_path(&self, path: PathBuf) -> Self { + Self { + flags: self.flags.clone(), + initial_cwd: self.initial_cwd.clone(), + maybe_node_modules_folder: Some(path), + maybe_vendor_folder: self.maybe_vendor_folder.clone(), + maybe_config_file: self.maybe_config_file.clone(), + maybe_package_json: self.maybe_package_json.clone(), + maybe_lockfile: self.maybe_lockfile.clone(), + overrides: self.overrides.clone(), + } + } + pub fn node_modules_dir_enablement(&self) -> Option { self.flags.node_modules_dir.or_else(|| { self diff --git a/cli/factory.rs b/cli/factory.rs index 8a58062c76..648d4a87d1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -29,15 +29,12 @@ use crate::module_loader::ModuleLoadPreparer; use crate::module_loader::NpmModuleLoader; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; -use crate::npm::create_npm_fs_resolver; -use crate::npm::CliNpmRegistryApi; +use crate::npm::create_cli_npm_resolver; use crate::npm::CliNpmResolver; -use crate::npm::ManagedCliNpmResolver; -use crate::npm::NpmCache; -use crate::npm::NpmCacheDir; -use crate::npm::NpmPackageFsResolver; -use crate::npm::NpmResolution; -use crate::npm::PackageJsonDepsInstaller; +use crate::npm::CliNpmResolverCreateOptions; +use crate::npm::CliNpmResolverManagedCreateOptions; +use crate::npm::CliNpmResolverManagedPackageJsonInstallerOption; +use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::standalone::DenoCompileBinaryWriter; @@ -157,12 +154,8 @@ struct CliFactoryServices { module_load_preparer: Deferred>, node_code_translator: Deferred>, node_resolver: Deferred>, - npm_api: Deferred>, - npm_cache: Deferred>, npm_resolver: Deferred>, - npm_resolution: Deferred>, package_json_deps_provider: Deferred>, - package_json_deps_installer: Deferred>, text_only_progress_bar: Deferred, type_checker: Deferred>, cjs_resolutions: Deferred>, @@ -294,47 +287,6 @@ impl CliFactory { .get_or_init(|| self.options.maybe_lockfile()) } - pub fn npm_cache(&self) -> Result<&Arc, AnyError> { - self.services.npm_cache.get_or_try_init(|| { - Ok(Arc::new(NpmCache::new( - NpmCacheDir::new(self.deno_dir()?.npm_folder_path()), - self.options.cache_setting(), - self.fs().clone(), - self.http_client().clone(), - self.text_only_progress_bar().clone(), - ))) - }) - } - - pub fn npm_api(&self) -> Result<&Arc, AnyError> { - self.services.npm_api.get_or_try_init(|| { - Ok(Arc::new(CliNpmRegistryApi::new( - CliNpmRegistryApi::default_url().to_owned(), - self.npm_cache()?.clone(), - self.http_client().clone(), - self.text_only_progress_bar().clone(), - ))) - }) - } - - pub async fn npm_resolution(&self) -> Result<&Arc, AnyError> { - self - .services - .npm_resolution - .get_or_try_init_async(async { - let npm_api = self.npm_api()?; - Ok(Arc::new(NpmResolution::from_serialized( - npm_api.clone(), - self - .options - .resolve_npm_resolution_snapshot(npm_api) - .await?, - self.maybe_lockfile().as_ref().cloned(), - ))) - }) - .await - } - pub async fn npm_resolver( &self, ) -> Result<&Arc, AnyError> { @@ -342,46 +294,39 @@ impl CliFactory { .services .npm_resolver .get_or_try_init_async(async { - let npm_resolution = self.npm_resolution().await?; - let fs = self.fs().clone(); - let npm_fs_resolver = create_npm_fs_resolver( - fs.clone(), - self.npm_cache()?.clone(), - self.text_only_progress_bar(), - CliNpmRegistryApi::default_url().to_owned(), - npm_resolution.clone(), - self.options.node_modules_dir_path(), - self.options.npm_system_info(), - ); - Ok(Arc::new(ManagedCliNpmResolver::new( - self.npm_api()?.clone(), - fs.clone(), - npm_resolution.clone(), - npm_fs_resolver, - self.maybe_lockfile().as_ref().cloned(), - self.package_json_deps_installer().await?.clone(), - )) as Arc) + create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { + snapshot: match self.options.resolve_npm_resolution_snapshot()? { + Some(snapshot) => { + CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)) + } + None => match self.maybe_lockfile() { + Some(lockfile) => { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( + lockfile.clone(), + ) + } + None => CliNpmResolverManagedSnapshotOption::Specified(None), + }, + }, + maybe_lockfile: self.maybe_lockfile().as_ref().cloned(), + fs: self.fs().clone(), + http_client: self.http_client().clone(), + npm_global_cache_dir: self.deno_dir()?.npm_folder_path(), + cache_setting: self.options.cache_setting(), + text_only_progress_bar: self.text_only_progress_bar().clone(), + maybe_node_modules_path: self.options.node_modules_dir_path(), + package_json_installer: + CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall( + self.package_json_deps_provider().clone(), + ), + npm_system_info: self.options.npm_system_info(), + npm_registry_url: crate::args::npm_registry_default_url().to_owned(), + })) + .await }) .await } - pub async fn create_node_modules_npm_fs_resolver( - &self, - node_modules_dir_path: PathBuf, - ) -> Result, AnyError> { - Ok(create_npm_fs_resolver( - self.fs().clone(), - self.npm_cache()?.clone(), - self.text_only_progress_bar(), - CliNpmRegistryApi::default_url().to_owned(), - self.npm_resolution().await?.clone(), - // when an explicit path is provided here, it will create the - // local node_modules variant of an npm fs resolver - Some(node_modules_dir_path), - self.options.npm_system_info(), - )) - } - pub fn package_json_deps_provider(&self) -> &Arc { self.services.package_json_deps_provider.get_or_init(|| { Arc::new(PackageJsonDepsProvider::new( @@ -390,22 +335,6 @@ impl CliFactory { }) } - pub async fn package_json_deps_installer( - &self, - ) -> Result<&Arc, AnyError> { - self - .services - .package_json_deps_installer - .get_or_try_init_async(async { - Ok(Arc::new(PackageJsonDepsInstaller::new( - self.package_json_deps_provider().clone(), - self.npm_api()?.clone(), - self.npm_resolution().await?.clone(), - ))) - }) - .await - } - pub async fn maybe_import_map( &self, ) -> Result<&Option>, AnyError> { @@ -616,9 +545,6 @@ impl CliFactory { self.file_fetcher()?, self.http_client(), self.deno_dir()?, - self.npm_api()?, - self.npm_cache()?, - self.npm_resolution().await?, self.npm_resolver().await?.as_ref(), self.options.npm_system_info(), self.package_json_deps_provider(), diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs index 8b2560f16c..6515e7dc08 100644 --- a/cli/lsp/analysis.rs +++ b/cli/lsp/analysis.rs @@ -161,14 +161,14 @@ fn code_as_string(code: &Option) -> String { pub struct TsResponseImportMapper<'a> { documents: &'a Documents, maybe_import_map: Option<&'a ImportMap>, - npm_resolver: &'a dyn CliNpmResolver, + npm_resolver: Option<&'a dyn CliNpmResolver>, } impl<'a> TsResponseImportMapper<'a> { pub fn new( documents: &'a Documents, maybe_import_map: Option<&'a ImportMap>, - npm_resolver: &'a dyn CliNpmResolver, + npm_resolver: Option<&'a dyn CliNpmResolver>, ) -> Self { Self { documents, @@ -194,7 +194,9 @@ impl<'a> TsResponseImportMapper<'a> { } } - if let Some(npm_resolver) = self.npm_resolver.as_managed() { + if let Some(npm_resolver) = + self.npm_resolver.as_ref().and_then(|r| r.as_managed()) + { if npm_resolver.in_npm_package(specifier) { if let Ok(Some(pkg_id)) = npm_resolver.resolve_pkg_id_from_specifier(specifier) @@ -250,8 +252,8 @@ impl<'a> TsResponseImportMapper<'a> { let specifier_path = specifier.to_file_path().ok()?; let root_folder = self .npm_resolver - .resolve_pkg_folder_from_specifier(specifier) - .ok() + .as_ref() + .and_then(|r| r.resolve_pkg_folder_from_specifier(specifier).ok()) .flatten()?; let package_json_path = root_folder.join("package.json"); let package_json_text = std::fs::read_to_string(&package_json_path).ok()?; diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 7bed7b1bff..dc36c05409 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -4,6 +4,7 @@ use deno_ast::MediaType; use deno_core::anyhow::anyhow; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; use deno_core::resolve_url; use deno_core::serde_json; use deno_core::serde_json::json; @@ -12,7 +13,6 @@ use deno_core::unsync::spawn; use deno_core::ModuleSpecifier; use deno_graph::GraphKind; use deno_lockfile::Lockfile; -use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmSystemInfo; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodeResolver; @@ -82,7 +82,6 @@ use super::urls::LspClientUrl; use crate::args::get_root_cert_store; use crate::args::package_json; use crate::args::resolve_import_map_from_specifier; -use crate::args::snapshot_from_lockfile; use crate::args::CaData; use crate::args::CacheSetting; use crate::args::CliOptions; @@ -102,14 +101,12 @@ use crate::graph_util; use crate::http_util::HttpClient; use crate::lsp::tsc::file_text_changes_to_workspace_edit; use crate::lsp::urls::LspUrlKind; -use crate::npm::create_npm_fs_resolver; -use crate::npm::CliNpmRegistryApi; +use crate::npm::create_cli_npm_resolver_for_lsp; use crate::npm::CliNpmResolver; -use crate::npm::ManagedCliNpmResolver; -use crate::npm::NpmCache; -use crate::npm::NpmCacheDir; -use crate::npm::NpmResolution; -use crate::npm::PackageJsonDepsInstaller; +use crate::npm::CliNpmResolverCreateOptions; +use crate::npm::CliNpmResolverManagedCreateOptions; +use crate::npm::CliNpmResolverManagedPackageJsonInstallerOption; +use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::tools::fmt::format_file; use crate::tools::fmt::format_parsed_source; use crate::util::fs::remove_dir_all_if_exists; @@ -130,16 +127,10 @@ impl RootCertStoreProvider for LspRootCertStoreProvider { struct LspNpmServices { /// When this hash changes, the services need updating config_hash: LspNpmConfigHash, - /// Npm's registry api. - api: Arc, /// Npm's search api. search_api: CliNpmSearchApi, - /// Npm cache - cache: Arc, - /// Npm resolution that is stored in memory. - resolution: Arc, /// Resolver for npm packages. - resolver: Arc, + resolver: Option>, } #[derive(Debug, PartialEq, Eq)] @@ -475,72 +466,6 @@ impl LanguageServer { } } -fn create_npm_api_and_cache( - dir: &DenoDir, - http_client: Arc, - registry_url: &ModuleSpecifier, - progress_bar: &ProgressBar, -) -> (Arc, Arc) { - let npm_cache = Arc::new(NpmCache::new( - NpmCacheDir::new(dir.npm_folder_path()), - // Use an "only" cache setting in order to make the - // user do an explicit "cache" command and prevent - // the cache from being filled with lots of packages while - // the user is typing. - CacheSetting::Only, - Arc::new(deno_fs::RealFs), - http_client.clone(), - progress_bar.clone(), - )); - let api = Arc::new(CliNpmRegistryApi::new( - registry_url.clone(), - npm_cache.clone(), - http_client, - progress_bar.clone(), - )); - (api, npm_cache) -} - -fn create_npm_resolver_and_resolution( - registry_url: &ModuleSpecifier, - progress_bar: ProgressBar, - api: Arc, - npm_cache: Arc, - node_modules_dir_path: Option, - maybe_snapshot: Option, -) -> (Arc, Arc) { - let resolution = Arc::new(NpmResolution::from_serialized( - api.clone(), - maybe_snapshot, - // Don't provide the lockfile. We don't want these resolvers - // updating it. Only the cache request should update the lockfile. - None, - )); - let fs = Arc::new(deno_fs::RealFs); - let fs_resolver = create_npm_fs_resolver( - fs.clone(), - npm_cache, - &progress_bar, - registry_url.clone(), - resolution.clone(), - node_modules_dir_path, - NpmSystemInfo::default(), - ); - ( - Arc::new(ManagedCliNpmResolver::new( - api, - fs, - resolution.clone(), - fs_resolver, - // Don't provide the lockfile. We don't want these resolvers - // updating it. Only the cache request should update the lockfile. - None, - Arc::new(PackageJsonDepsInstaller::no_op()), - )), - resolution, - ) -} - impl Inner { fn new(client: Client) -> Self { let dir = DenoDir::new(None).expect("could not access DENO_DIR"); @@ -571,23 +496,6 @@ impl Inner { diagnostics_state.clone(), ); let assets = Assets::new(ts_server.clone()); - let registry_url = CliNpmRegistryApi::default_url(); - let progress_bar = ProgressBar::new(ProgressBarStyle::TextOnly); - - let (npm_api, npm_cache) = create_npm_api_and_cache( - &dir, - http_client.clone(), - registry_url, - &progress_bar, - ); - let (npm_resolver, npm_resolution) = create_npm_resolver_and_resolution( - registry_url, - progress_bar, - npm_api.clone(), - npm_cache.clone(), - None, - None, - ); Self { assets, @@ -610,11 +518,8 @@ impl Inner { module_registries_location, npm: LspNpmServices { config_hash: LspNpmConfigHash(0), // this will be updated in initialize - api: npm_api, search_api: npm_search_api, - cache: npm_cache, - resolution: npm_resolution, - resolver: npm_resolver, + resolver: None, }, performance, ts_fixable_diagnostics: Default::default(), @@ -799,41 +704,27 @@ impl Inner { } pub fn snapshot(&self) -> Arc { - // create a new snapshotted npm resolution and resolver - let npm_resolution = Arc::new(NpmResolution::new( - self.npm.api.clone(), - self.npm.resolution.snapshot(), - self.config.maybe_lockfile().cloned(), - )); - let node_fs = Arc::new(deno_fs::RealFs); - let npm_resolver = Arc::new(ManagedCliNpmResolver::new( - self.npm.api.clone(), - node_fs.clone(), - npm_resolution.clone(), - create_npm_fs_resolver( - node_fs.clone(), - self.npm.cache.clone(), - &ProgressBar::new(ProgressBarStyle::TextOnly), - self.npm.api.base_url().clone(), - npm_resolution, - self.config.maybe_node_modules_dir_path().cloned(), - NpmSystemInfo::default(), - ), - self.config.maybe_lockfile().cloned(), - Arc::new(PackageJsonDepsInstaller::no_op()), - )); - let node_resolver = - Arc::new(NodeResolver::new(node_fs, npm_resolver.clone())); + let maybe_state_npm_snapshot = self + .npm + .resolver + .as_ref() + .map(|resolver| resolver.clone_snapshotted()) + .map(|resolver| { + let fs = Arc::new(deno_fs::RealFs); + let node_resolver = + Arc::new(NodeResolver::new(fs, resolver.clone().into_npm_resolver())); + StateNpmSnapshot { + node_resolver, + npm_resolver: resolver, + } + }); Arc::new(StateSnapshot { assets: self.assets.snapshot(), cache_metadata: self.cache_metadata.clone(), config: self.config.snapshot(), documents: self.documents.clone(), maybe_import_map: self.maybe_import_map.clone(), - npm: Some(StateNpmSnapshot { - node_resolver, - npm_resolver, - }), + npm: maybe_state_npm_snapshot, }) } @@ -935,25 +826,6 @@ impl Inner { Ok(()) } - async fn get_npm_snapshot( - &self, - ) -> Option { - let lockfile = self.config.maybe_lockfile()?; - let snapshot = - match snapshot_from_lockfile(lockfile.clone(), &*self.npm.api).await { - Ok(snapshot) => snapshot, - Err(err) => { - lsp_warn!("Failed getting npm snapshot from lockfile: {}", err); - return None; - } - }; - - // clear the memory cache to reduce memory usage - self.npm.api.clear_memory_cache(); - - Some(snapshot) - } - async fn recreate_npm_services_if_necessary(&mut self) { let deno_dir = match DenoDir::new(self.maybe_global_cache_path.clone()) { Ok(deno_dir) => deno_dir, @@ -967,24 +839,15 @@ impl Inner { return; // no need to do anything } - let registry_url = CliNpmRegistryApi::default_url(); - let progress_bar = ProgressBar::new(ProgressBarStyle::TextOnly); - (self.npm.api, self.npm.cache) = create_npm_api_and_cache( - &deno_dir, - self.http_client.clone(), - registry_url, - &progress_bar, - ); - let maybe_snapshot = self.get_npm_snapshot().await; - (self.npm.resolver, self.npm.resolution) = - create_npm_resolver_and_resolution( - registry_url, - progress_bar, - self.npm.api.clone(), - self.npm.cache.clone(), + self.npm.resolver = Some( + create_npm_resolver( + &deno_dir, + &self.http_client, + self.config.maybe_lockfile(), self.config.maybe_node_modules_dir_path().cloned(), - maybe_snapshot, - ); + ) + .await, + ); // update the hash self.npm.config_hash = config_hash; @@ -1217,6 +1080,45 @@ impl Inner { } } +async fn create_npm_resolver( + deno_dir: &DenoDir, + http_client: &Arc, + maybe_lockfile: Option<&Arc>>, + maybe_node_modules_dir_path: Option, +) -> Arc { + create_cli_npm_resolver_for_lsp(CliNpmResolverCreateOptions::Managed( + CliNpmResolverManagedCreateOptions { + http_client: http_client.clone(), + snapshot: match maybe_lockfile { + Some(lockfile) => { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile( + lockfile.clone(), + ) + } + None => CliNpmResolverManagedSnapshotOption::Specified(None), + }, + // Don't provide the lockfile. We don't want these resolvers + // updating it. Only the cache request should update the lockfile. + maybe_lockfile: None, + fs: Arc::new(deno_fs::RealFs), + npm_global_cache_dir: deno_dir.npm_folder_path(), + // Use an "only" cache setting in order to make the + // user do an explicit "cache" command and prevent + // the cache from being filled with lots of packages while + // the user is typing. + cache_setting: CacheSetting::Only, + text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly), + maybe_node_modules_path: maybe_node_modules_dir_path, + // do not install while resolving in the lsp—leave that to the cache command + package_json_installer: + CliNpmResolverManagedPackageJsonInstallerOption::NoInstall, + npm_registry_url: crate::args::npm_registry_default_url().to_owned(), + npm_system_info: NpmSystemInfo::default(), + }, + )) + .await +} + // lspower::LanguageServer methods. This file's LanguageServer delegates to us. impl Inner { async fn initialize( @@ -1371,7 +1273,7 @@ impl Inner { maybe_import_map: self.maybe_import_map.clone(), maybe_config_file: self.config.maybe_config_file(), maybe_package_json: self.maybe_package_json.as_ref(), - npm_resolver: Some(self.npm.resolver.clone()), + npm_resolver: self.npm.resolver.clone(), }); // refresh the npm specifiers because it might have discovered @@ -1446,7 +1348,9 @@ impl Inner { let npm_resolver = self.npm.resolver.clone(); // spawn to avoid the LSP's Send requirements let handle = spawn(async move { - if let Some(npm_resolver) = npm_resolver.as_managed() { + if let Some(npm_resolver) = + npm_resolver.as_ref().and_then(|r| r.as_managed()) + { npm_resolver.set_package_reqs(&package_reqs).await } else { Ok(()) @@ -2165,7 +2069,7 @@ impl Inner { TsResponseImportMapper::new( &self.documents, self.maybe_import_map.as_deref(), - self.npm.resolver.as_ref(), + self.npm.resolver.as_deref(), ) } diff --git a/cli/lsp/npm.rs b/cli/lsp/npm.rs index 0f2794e44d..223bfa61da 100644 --- a/cli/lsp/npm.rs +++ b/cli/lsp/npm.rs @@ -12,8 +12,8 @@ use deno_npm::registry::NpmPackageInfo; use deno_runtime::permissions::PermissionsContainer; use serde::Deserialize; +use crate::args::npm_registry_default_url; use crate::file_fetcher::FileFetcher; -use crate::npm::CliNpmRegistryApi; #[async_trait::async_trait] pub trait NpmSearchApi { @@ -36,7 +36,7 @@ impl CliNpmSearchApi { pub fn new(file_fetcher: FileFetcher, custom_base_url: Option) -> Self { Self { base_url: custom_base_url - .unwrap_or_else(|| CliNpmRegistryApi::default_url().clone()), + .unwrap_or_else(|| npm_registry_default_url().clone()), file_fetcher, info_cache: Default::default(), search_cache: Default::default(), diff --git a/cli/npm/cache_dir.rs b/cli/npm/cache_dir.rs new file mode 100644 index 0000000000..b0d0490470 --- /dev/null +++ b/cli/npm/cache_dir.rs @@ -0,0 +1,268 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; +use std::path::PathBuf; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::url::Url; +use deno_npm::NpmPackageCacheFolderId; +use deno_semver::package::PackageNv; +use deno_semver::Version; + +use crate::util::fs::canonicalize_path; +use crate::util::path::root_url_to_safe_local_dirname; + +/// The global cache directory of npm packages. +#[derive(Clone, Debug)] +pub struct NpmCacheDir { + root_dir: PathBuf, + // cached url representation of the root directory + root_dir_url: Url, +} + +impl NpmCacheDir { + pub fn new(root_dir: PathBuf) -> Self { + fn try_get_canonicalized_root_dir( + root_dir: &Path, + ) -> Result { + if !root_dir.exists() { + std::fs::create_dir_all(root_dir) + .with_context(|| format!("Error creating {}", root_dir.display()))?; + } + Ok(canonicalize_path(root_dir)?) + } + + // this may fail on readonly file systems, so just ignore if so + let root_dir = + try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir); + let root_dir_url = Url::from_directory_path(&root_dir).unwrap(); + Self { + root_dir, + root_dir_url, + } + } + + pub fn root_dir_url(&self) -> &Url { + &self.root_dir_url + } + + pub fn package_folder_for_id( + &self, + folder_id: &NpmPackageCacheFolderId, + registry_url: &Url, + ) -> PathBuf { + if folder_id.copy_index == 0 { + self.package_folder_for_name_and_version(&folder_id.nv, registry_url) + } else { + self + .package_name_folder(&folder_id.nv.name, registry_url) + .join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index)) + } + } + + pub fn package_folder_for_name_and_version( + &self, + package: &PackageNv, + registry_url: &Url, + ) -> PathBuf { + self + .package_name_folder(&package.name, registry_url) + .join(package.version.to_string()) + } + + pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { + let mut dir = self.registry_folder(registry_url); + if name.to_lowercase() != name { + let encoded_name = mixed_case_package_name_encode(name); + // Using the encoded directory may have a collision with an actual package name + // so prefix it with an underscore since npm packages can't start with that + dir.join(format!("_{encoded_name}")) + } else { + // ensure backslashes are used on windows + for part in name.split('/') { + dir = dir.join(part); + } + dir + } + } + + pub fn registry_folder(&self, registry_url: &Url) -> PathBuf { + self + .root_dir + .join(root_url_to_safe_local_dirname(registry_url)) + } + + pub fn resolve_package_folder_id_from_specifier( + &self, + specifier: &ModuleSpecifier, + registry_url: &Url, + ) -> Option { + let registry_root_dir = self + .root_dir_url + .join(&format!( + "{}/", + root_url_to_safe_local_dirname(registry_url) + .to_string_lossy() + .replace('\\', "/") + )) + // this not succeeding indicates a fatal issue, so unwrap + .unwrap(); + let mut relative_url = registry_root_dir.make_relative(specifier)?; + if relative_url.starts_with("../") { + return None; + } + + // base32 decode the url if it starts with an underscore + // * Ex. _{base32(package_name)}/ + if let Some(end_url) = relative_url.strip_prefix('_') { + let mut parts = end_url + .split('/') + .map(ToOwned::to_owned) + .collect::>(); + match mixed_case_package_name_decode(&parts[0]) { + Some(part) => { + parts[0] = part; + } + None => return None, + } + relative_url = parts.join("/"); + } + + // examples: + // * chalk/5.0.1/ + // * @types/chalk/5.0.1/ + // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps + let is_scoped_package = relative_url.starts_with('@'); + let mut parts = relative_url + .split('/') + .enumerate() + .take(if is_scoped_package { 3 } else { 2 }) + .map(|(_, part)| part) + .collect::>(); + if parts.len() < 2 { + return None; + } + let version_part = parts.pop().unwrap(); + let name = parts.join("/"); + let (version, copy_index) = + if let Some((version, copy_count)) = version_part.split_once('_') { + (version, copy_count.parse::().ok()?) + } else { + (version_part, 0) + }; + Some(NpmPackageCacheFolderId { + nv: PackageNv { + name, + version: Version::parse_from_npm(version).ok()?, + }, + copy_index, + }) + } + + pub fn get_cache_location(&self) -> PathBuf { + self.root_dir.clone() + } +} + +pub fn mixed_case_package_name_encode(name: &str) -> String { + // use base32 encoding because it's reversible and the character set + // only includes the characters within 0-9 and A-Z so it can be lower cased + base32::encode( + base32::Alphabet::RFC4648 { padding: false }, + name.as_bytes(), + ) + .to_lowercase() +} + +pub fn mixed_case_package_name_decode(name: &str) -> Option { + base32::decode(base32::Alphabet::RFC4648 { padding: false }, name) + .and_then(|b| String::from_utf8(b).ok()) +} + +#[cfg(test)] +mod test { + use deno_core::url::Url; + use deno_semver::package::PackageNv; + use deno_semver::Version; + + use super::NpmCacheDir; + use crate::npm::cache_dir::NpmPackageCacheFolderId; + + #[test] + fn should_get_package_folder() { + let deno_dir = crate::cache::DenoDir::new(None).unwrap(); + let root_dir = deno_dir.npm_folder_path(); + let cache = NpmCacheDir::new(root_dir.clone()); + let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); + + assert_eq!( + cache.package_folder_for_id( + &NpmPackageCacheFolderId { + nv: PackageNv { + name: "json".to_string(), + version: Version::parse_from_npm("1.2.5").unwrap(), + }, + copy_index: 0, + }, + ®istry_url, + ), + root_dir + .join("registry.npmjs.org") + .join("json") + .join("1.2.5"), + ); + + assert_eq!( + cache.package_folder_for_id( + &NpmPackageCacheFolderId { + nv: PackageNv { + name: "json".to_string(), + version: Version::parse_from_npm("1.2.5").unwrap(), + }, + copy_index: 1, + }, + ®istry_url, + ), + root_dir + .join("registry.npmjs.org") + .join("json") + .join("1.2.5_1"), + ); + + assert_eq!( + cache.package_folder_for_id( + &NpmPackageCacheFolderId { + nv: PackageNv { + name: "JSON".to_string(), + version: Version::parse_from_npm("2.1.5").unwrap(), + }, + copy_index: 0, + }, + ®istry_url, + ), + root_dir + .join("registry.npmjs.org") + .join("_jjju6tq") + .join("2.1.5"), + ); + + assert_eq!( + cache.package_folder_for_id( + &NpmPackageCacheFolderId { + nv: PackageNv { + name: "@types/JSON".to_string(), + version: Version::parse_from_npm("2.1.5").unwrap(), + }, + copy_index: 0, + }, + ®istry_url, + ), + root_dir + .join("registry.npmjs.org") + .join("_ib2hs4dfomxuuu2pjy") + .join("2.1.5"), + ); + } +} diff --git a/cli/npm/cache.rs b/cli/npm/managed/cache.rs similarity index 54% rename from cli/npm/cache.rs rename to cli/npm/managed/cache.rs index f76bf6821e..91d6ec656b 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/managed/cache.rs @@ -17,241 +17,15 @@ use deno_npm::registry::NpmPackageVersionDistInfo; use deno_npm::NpmPackageCacheFolderId; use deno_runtime::deno_fs; use deno_semver::package::PackageNv; -use deno_semver::Version; use crate::args::CacheSetting; use crate::http_util::HttpClient; -use crate::util::fs::canonicalize_path; +use crate::npm::NpmCacheDir; use crate::util::fs::hard_link_dir_recursive; -use crate::util::path::root_url_to_safe_local_dirname; use crate::util::progress_bar::ProgressBar; use super::tarball::verify_and_extract_tarball; -const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; - -pub fn with_folder_sync_lock( - package: &PackageNv, - output_folder: &Path, - action: impl FnOnce() -> Result<(), AnyError>, -) -> Result<(), AnyError> { - fn inner( - output_folder: &Path, - action: impl FnOnce() -> Result<(), AnyError>, - ) -> Result<(), AnyError> { - fs::create_dir_all(output_folder).with_context(|| { - format!("Error creating '{}'.", output_folder.display()) - })?; - - // This sync lock file is a way to ensure that partially created - // npm package directories aren't considered valid. This could maybe - // be a bit smarter in the future to not bother extracting here - // if another process has taken the lock in the past X seconds and - // wait for the other process to finish (it could try to create the - // file with `create_new(true)` then if it exists, check the metadata - // then wait until the other process finishes with a timeout), but - // for now this is good enough. - let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME); - match fs::OpenOptions::new() - .write(true) - .create(true) - .open(&sync_lock_path) - { - Ok(_) => { - action()?; - // extraction succeeded, so only now delete this file - let _ignore = std::fs::remove_file(&sync_lock_path); - Ok(()) - } - Err(err) => { - bail!( - concat!( - "Error creating package sync lock file at '{}'. ", - "Maybe try manually deleting this folder.\n\n{:#}", - ), - output_folder.display(), - err - ); - } - } - } - - match inner(output_folder, action) { - Ok(()) => Ok(()), - Err(err) => { - if let Err(remove_err) = fs::remove_dir_all(output_folder) { - if remove_err.kind() != std::io::ErrorKind::NotFound { - bail!( - concat!( - "Failed setting up package cache directory for {}, then ", - "failed cleaning it up.\n\nOriginal error:\n\n{}\n\n", - "Remove error:\n\n{}\n\nPlease manually ", - "delete this folder or you will run into issues using this ", - "package in the future:\n\n{}" - ), - package, - err, - remove_err, - output_folder.display(), - ); - } - } - Err(err) - } - } -} - -#[derive(Clone, Debug)] -pub struct NpmCacheDir { - root_dir: PathBuf, - // cached url representation of the root directory - root_dir_url: Url, -} - -impl NpmCacheDir { - pub fn new(root_dir: PathBuf) -> Self { - fn try_get_canonicalized_root_dir( - root_dir: &Path, - ) -> Result { - if !root_dir.exists() { - std::fs::create_dir_all(root_dir) - .with_context(|| format!("Error creating {}", root_dir.display()))?; - } - Ok(canonicalize_path(root_dir)?) - } - - // this may fail on readonly file systems, so just ignore if so - let root_dir = - try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir); - let root_dir_url = Url::from_directory_path(&root_dir).unwrap(); - Self { - root_dir, - root_dir_url, - } - } - - pub fn root_dir_url(&self) -> &Url { - &self.root_dir_url - } - - pub fn package_folder_for_id( - &self, - folder_id: &NpmPackageCacheFolderId, - registry_url: &Url, - ) -> PathBuf { - if folder_id.copy_index == 0 { - self.package_folder_for_name_and_version(&folder_id.nv, registry_url) - } else { - self - .package_name_folder(&folder_id.nv.name, registry_url) - .join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index)) - } - } - - pub fn package_folder_for_name_and_version( - &self, - package: &PackageNv, - registry_url: &Url, - ) -> PathBuf { - self - .package_name_folder(&package.name, registry_url) - .join(package.version.to_string()) - } - - pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { - let mut dir = self.registry_folder(registry_url); - if name.to_lowercase() != name { - let encoded_name = mixed_case_package_name_encode(name); - // Using the encoded directory may have a collision with an actual package name - // so prefix it with an underscore since npm packages can't start with that - dir.join(format!("_{encoded_name}")) - } else { - // ensure backslashes are used on windows - for part in name.split('/') { - dir = dir.join(part); - } - dir - } - } - - pub fn registry_folder(&self, registry_url: &Url) -> PathBuf { - self - .root_dir - .join(root_url_to_safe_local_dirname(registry_url)) - } - - pub fn resolve_package_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - registry_url: &Url, - ) -> Option { - let registry_root_dir = self - .root_dir_url - .join(&format!( - "{}/", - root_url_to_safe_local_dirname(registry_url) - .to_string_lossy() - .replace('\\', "/") - )) - // this not succeeding indicates a fatal issue, so unwrap - .unwrap(); - let mut relative_url = registry_root_dir.make_relative(specifier)?; - if relative_url.starts_with("../") { - return None; - } - - // base32 decode the url if it starts with an underscore - // * Ex. _{base32(package_name)}/ - if let Some(end_url) = relative_url.strip_prefix('_') { - let mut parts = end_url - .split('/') - .map(ToOwned::to_owned) - .collect::>(); - match mixed_case_package_name_decode(&parts[0]) { - Some(part) => { - parts[0] = part; - } - None => return None, - } - relative_url = parts.join("/"); - } - - // examples: - // * chalk/5.0.1/ - // * @types/chalk/5.0.1/ - // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps - let is_scoped_package = relative_url.starts_with('@'); - let mut parts = relative_url - .split('/') - .enumerate() - .take(if is_scoped_package { 3 } else { 2 }) - .map(|(_, part)| part) - .collect::>(); - if parts.len() < 2 { - return None; - } - let version_part = parts.pop().unwrap(); - let name = parts.join("/"); - let (version, copy_index) = - if let Some((version, copy_count)) = version_part.split_once('_') { - (version, copy_count.parse::().ok()?) - } else { - (version_part, 0) - }; - Some(NpmPackageCacheFolderId { - nv: PackageNv { - name, - version: Version::parse_from_npm(version).ok()?, - }, - copy_index, - }) - } - - pub fn get_cache_location(&self) -> PathBuf { - self.root_dir.clone() - } -} - /// Stores a single copy of npm packages in a cache. #[derive(Debug)] pub struct NpmCache { @@ -282,10 +56,6 @@ impl NpmCache { } } - pub fn as_readonly(&self) -> NpmCacheDir { - self.cache_dir.clone() - } - pub fn cache_setting(&self) -> &CacheSetting { &self.cache_setting } @@ -434,103 +204,75 @@ impl NpmCache { } } -pub fn mixed_case_package_name_encode(name: &str) -> String { - // use base32 encoding because it's reversible and the character set - // only includes the characters within 0-9 and A-Z so it can be lower cased - base32::encode( - base32::Alphabet::RFC4648 { padding: false }, - name.as_bytes(), - ) - .to_lowercase() -} +const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; -pub fn mixed_case_package_name_decode(name: &str) -> Option { - base32::decode(base32::Alphabet::RFC4648 { padding: false }, name) - .and_then(|b| String::from_utf8(b).ok()) -} +pub fn with_folder_sync_lock( + package: &PackageNv, + output_folder: &Path, + action: impl FnOnce() -> Result<(), AnyError>, +) -> Result<(), AnyError> { + fn inner( + output_folder: &Path, + action: impl FnOnce() -> Result<(), AnyError>, + ) -> Result<(), AnyError> { + fs::create_dir_all(output_folder).with_context(|| { + format!("Error creating '{}'.", output_folder.display()) + })?; -#[cfg(test)] -mod test { - use deno_core::url::Url; - use deno_semver::package::PackageNv; - use deno_semver::Version; + // This sync lock file is a way to ensure that partially created + // npm package directories aren't considered valid. This could maybe + // be a bit smarter in the future to not bother extracting here + // if another process has taken the lock in the past X seconds and + // wait for the other process to finish (it could try to create the + // file with `create_new(true)` then if it exists, check the metadata + // then wait until the other process finishes with a timeout), but + // for now this is good enough. + let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME); + match fs::OpenOptions::new() + .write(true) + .create(true) + .open(&sync_lock_path) + { + Ok(_) => { + action()?; + // extraction succeeded, so only now delete this file + let _ignore = std::fs::remove_file(&sync_lock_path); + Ok(()) + } + Err(err) => { + bail!( + concat!( + "Error creating package sync lock file at '{}'. ", + "Maybe try manually deleting this folder.\n\n{:#}", + ), + output_folder.display(), + err + ); + } + } + } - use super::NpmCacheDir; - use crate::npm::cache::NpmPackageCacheFolderId; - - #[test] - fn should_get_package_folder() { - let deno_dir = crate::cache::DenoDir::new(None).unwrap(); - let root_dir = deno_dir.npm_folder_path(); - let cache = NpmCacheDir::new(root_dir.clone()); - let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "json".to_string(), - version: Version::parse_from_npm("1.2.5").unwrap(), - }, - copy_index: 1, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("json") - .join("1.2.5_1"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_jjju6tq") - .join("2.1.5"), - ); - - assert_eq!( - cache.package_folder_for_id( - &NpmPackageCacheFolderId { - nv: PackageNv { - name: "@types/JSON".to_string(), - version: Version::parse_from_npm("2.1.5").unwrap(), - }, - copy_index: 0, - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_ib2hs4dfomxuuu2pjy") - .join("2.1.5"), - ); + match inner(output_folder, action) { + Ok(()) => Ok(()), + Err(err) => { + if let Err(remove_err) = fs::remove_dir_all(output_folder) { + if remove_err.kind() != std::io::ErrorKind::NotFound { + bail!( + concat!( + "Failed setting up package cache directory for {}, then ", + "failed cleaning it up.\n\nOriginal error:\n\n{}\n\n", + "Remove error:\n\n{}\n\nPlease manually ", + "delete this folder or you will run into issues using this ", + "package in the future:\n\n{}" + ), + package, + err, + remove_err, + output_folder.display(), + ); + } + } + Err(err) + } } } diff --git a/cli/npm/managed/installer.rs b/cli/npm/managed/installer.rs index 21285c3d7f..8f3db05319 100644 --- a/cli/npm/managed/installer.rs +++ b/cli/npm/managed/installer.rs @@ -13,7 +13,7 @@ use deno_semver::package::PackageReq; use crate::args::PackageJsonDepsProvider; use crate::util::sync::AtomicFlag; -use super::super::CliNpmRegistryApi; +use super::CliNpmRegistryApi; use super::NpmResolution; #[derive(Debug)] diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index c5ba3d3aff..df9ad59acd 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -6,6 +6,7 @@ use std::path::PathBuf; use std::sync::Arc; use deno_ast::ModuleSpecifier; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::serde_json; @@ -14,7 +15,7 @@ use deno_graph::NpmPackageReqResolution; use deno_npm::registry::NpmRegistryApi; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::resolution::PackageReqNotFoundError; -use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; @@ -27,30 +28,213 @@ use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageNvReference; use deno_semver::package::PackageReq; -use serde::Deserialize; -use serde::Serialize; use crate::args::Lockfile; +use crate::args::NpmProcessState; +use crate::args::PackageJsonDepsProvider; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::util::progress_bar::ProgressBar; + +use self::cache::NpmCache; +use self::installer::PackageJsonDepsInstaller; +use self::registry::CliNpmRegistryApi; +use self::resolution::NpmResolution; +use self::resolvers::create_npm_fs_resolver; +use self::resolvers::NpmPackageFsResolver; -use super::CliNpmRegistryApi; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; +use super::NpmCacheDir; -pub use self::installer::PackageJsonDepsInstaller; -pub use self::resolution::NpmResolution; -pub use self::resolvers::create_npm_fs_resolver; -pub use self::resolvers::NpmPackageFsResolver; - +mod cache; mod installer; +mod registry; mod resolution; mod resolvers; +mod tarball; -/// State provided to the process via an environment variable. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct NpmProcessState { - pub snapshot: SerializedNpmResolutionSnapshot, - pub local_node_modules_path: Option, +pub enum CliNpmResolverManagedSnapshotOption { + ResolveFromLockfile(Arc>), + Specified(Option), +} + +pub enum CliNpmResolverManagedPackageJsonInstallerOption { + ConditionalInstall(Arc), + NoInstall, +} + +pub struct CliNpmResolverManagedCreateOptions { + pub snapshot: CliNpmResolverManagedSnapshotOption, + pub maybe_lockfile: Option>>, + pub fs: Arc, + pub http_client: Arc, + pub npm_global_cache_dir: PathBuf, + pub cache_setting: crate::args::CacheSetting, + pub text_only_progress_bar: crate::util::progress_bar::ProgressBar, + pub maybe_node_modules_path: Option, + pub npm_system_info: NpmSystemInfo, + pub package_json_installer: CliNpmResolverManagedPackageJsonInstallerOption, + pub npm_registry_url: Url, +} + +pub async fn create_managed_npm_resolver_for_lsp( + options: CliNpmResolverManagedCreateOptions, +) -> Arc { + let npm_cache = create_cache(&options); + let npm_api = create_api(&options, npm_cache.clone()); + let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await { + Ok(snapshot) => snapshot, + Err(err) => { + log::warn!("failed to resolve snapshot: {}", err); + None + } + }; + create_inner( + npm_cache, + npm_api, + snapshot, + options.maybe_lockfile, + options.fs, + options.text_only_progress_bar, + options.maybe_node_modules_path, + options.package_json_installer, + options.npm_registry_url, + options.npm_system_info, + ) +} + +pub async fn create_managed_npm_resolver( + options: CliNpmResolverManagedCreateOptions, +) -> Result, AnyError> { + let npm_cache = create_cache(&options); + let npm_api = create_api(&options, npm_cache.clone()); + let snapshot = resolve_snapshot(&npm_api, options.snapshot).await?; + Ok(create_inner( + npm_cache, + npm_api, + snapshot, + options.maybe_lockfile, + options.fs, + options.text_only_progress_bar, + options.maybe_node_modules_path, + options.package_json_installer, + options.npm_registry_url, + options.npm_system_info, + )) +} + +#[allow(clippy::too_many_arguments)] +fn create_inner( + npm_cache: Arc, + npm_api: Arc, + snapshot: Option, + maybe_lockfile: Option>>, + fs: Arc, + text_only_progress_bar: crate::util::progress_bar::ProgressBar, + node_modules_dir_path: Option, + package_json_installer: CliNpmResolverManagedPackageJsonInstallerOption, + npm_registry_url: Url, + npm_system_info: NpmSystemInfo, +) -> Arc { + let resolution = Arc::new(NpmResolution::from_serialized( + npm_api.clone(), + snapshot, + maybe_lockfile.clone(), + )); + let npm_fs_resolver = create_npm_fs_resolver( + fs.clone(), + npm_cache.clone(), + &text_only_progress_bar, + npm_registry_url, + resolution.clone(), + node_modules_dir_path, + npm_system_info.clone(), + ); + let package_json_deps_installer = match package_json_installer { + CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall( + provider, + ) => Arc::new(PackageJsonDepsInstaller::new( + provider, + npm_api.clone(), + resolution.clone(), + )), + CliNpmResolverManagedPackageJsonInstallerOption::NoInstall => { + Arc::new(PackageJsonDepsInstaller::no_op()) + } + }; + Arc::new(ManagedCliNpmResolver::new( + npm_api, + fs, + resolution, + npm_fs_resolver, + npm_cache, + maybe_lockfile, + package_json_deps_installer, + text_only_progress_bar, + npm_system_info, + )) +} + +fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc { + Arc::new(NpmCache::new( + NpmCacheDir::new(options.npm_global_cache_dir.clone()), + options.cache_setting.clone(), + options.fs.clone(), + options.http_client.clone(), + options.text_only_progress_bar.clone(), + )) +} + +fn create_api( + options: &CliNpmResolverManagedCreateOptions, + npm_cache: Arc, +) -> Arc { + Arc::new(CliNpmRegistryApi::new( + options.npm_registry_url.clone(), + npm_cache.clone(), + options.http_client.clone(), + options.text_only_progress_bar.clone(), + )) +} + +async fn resolve_snapshot( + api: &CliNpmRegistryApi, + snapshot: CliNpmResolverManagedSnapshotOption, +) -> Result, AnyError> { + match snapshot { + CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(lockfile) => { + if !lockfile.lock().overwrite { + let snapshot = snapshot_from_lockfile(lockfile.clone(), api) + .await + .with_context(|| { + format!( + "failed reading lockfile '{}'", + lockfile.lock().filename.display() + ) + })?; + // clear the memory cache to reduce memory usage + api.clear_memory_cache(); + Ok(Some(snapshot)) + } else { + Ok(None) + } + } + CliNpmResolverManagedSnapshotOption::Specified(snapshot) => Ok(snapshot), + } +} + +async fn snapshot_from_lockfile( + lockfile: Arc>, + api: &dyn NpmRegistryApi, +) -> Result { + let incomplete_snapshot = { + let lock = lockfile.lock(); + deno_npm::resolution::incomplete_snapshot_from_lockfile(&lock)? + }; + let snapshot = + deno_npm::resolution::snapshot_from_lockfile(incomplete_snapshot, api) + .await?; + Ok(snapshot) } /// An npm resolver where the resolution is managed by Deno rather than @@ -59,40 +243,45 @@ pub struct ManagedCliNpmResolver { api: Arc, fs: Arc, fs_resolver: Arc, + global_npm_cache: Arc, resolution: Arc, maybe_lockfile: Option>>, + npm_system_info: NpmSystemInfo, + progress_bar: ProgressBar, package_json_deps_installer: Arc, } impl std::fmt::Debug for ManagedCliNpmResolver { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ManagedNpmResolver") - .field("api", &"") - .field("fs", &"") - .field("fs_resolver", &"") - .field("resolution", &"") - .field("maybe_lockfile", &"") - .field("package_json_deps_installer", &"") + .field("", &"") .finish() } } impl ManagedCliNpmResolver { + #[allow(clippy::too_many_arguments)] pub fn new( api: Arc, fs: Arc, resolution: Arc, fs_resolver: Arc, + global_npm_cache: Arc, maybe_lockfile: Option>>, package_json_deps_installer: Arc, + progress_bar: ProgressBar, + npm_system_info: NpmSystemInfo, ) -> Self { Self { api, fs, fs_resolver, + global_npm_cache, resolution, maybe_lockfile, package_json_deps_installer, + progress_bar, + npm_system_info, } } @@ -191,6 +380,15 @@ impl ManagedCliNpmResolver { self.resolution.snapshot() } + pub fn serialized_valid_snapshot_for_system( + &self, + system_info: &NpmSystemInfo, + ) -> ValidSerializedNpmResolutionSnapshot { + self + .resolution + .serialized_valid_snapshot_for_system(system_info) + } + pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { self.resolution.lock(lockfile) } @@ -208,8 +406,11 @@ impl ManagedCliNpmResolver { pub async fn resolve_pending(&self) -> Result<(), AnyError> { self.resolution.resolve_pending().await?; - self.fs_resolver.cache_packages().await?; - Ok(()) + self.cache_packages().await + } + + pub async fn cache_packages(&self) -> Result<(), AnyError> { + self.fs_resolver.cache_packages().await } fn resolve_pkg_id_from_pkg_req( @@ -240,6 +441,17 @@ impl ManagedCliNpmResolver { .map(|_| ()) .map_err(|err| err.into()) } + + pub fn registry_base_url(&self) -> &ModuleSpecifier { + self.api.base_url() + } + + pub fn registry_folder_in_global_cache( + &self, + registry_url: &ModuleSpecifier, + ) -> PathBuf { + self.global_npm_cache.registry_folder(registry_url) + } } impl NpmResolver for ManagedCliNpmResolver { @@ -283,6 +495,35 @@ impl CliNpmResolver for ManagedCliNpmResolver { self } + fn clone_snapshotted(&self) -> Arc { + // create a new snapshotted npm resolution and resolver + let npm_resolution = Arc::new(NpmResolution::new( + self.api.clone(), + self.resolution.snapshot(), + self.maybe_lockfile.clone(), + )); + + Arc::new(ManagedCliNpmResolver::new( + self.api.clone(), + self.fs.clone(), + npm_resolution.clone(), + create_npm_fs_resolver( + self.fs.clone(), + self.global_npm_cache.clone(), + &self.progress_bar, + self.api.base_url().clone(), + npm_resolution, + self.node_modules_path(), + self.npm_system_info.clone(), + ), + self.global_npm_cache.clone(), + self.maybe_lockfile.clone(), + self.package_json_deps_installer.clone(), + self.progress_bar.clone(), + self.npm_system_info.clone(), + )) + } + fn root_dir_url(&self) -> &Url { self.fs_resolver.root_dir_url() } diff --git a/cli/npm/registry.rs b/cli/npm/managed/registry.rs similarity index 93% rename from cli/npm/registry.rs rename to cli/npm/managed/registry.rs index 61eb4123dd..2466f47131 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/managed/registry.rs @@ -21,7 +21,6 @@ use deno_core::url::Url; use deno_npm::registry::NpmPackageInfo; use deno_npm::registry::NpmRegistryApi; use deno_npm::registry::NpmRegistryPackageInfoLoadError; -use once_cell::sync::Lazy; use crate::args::CacheSetting; use crate::cache::CACHE_PERM; @@ -32,32 +31,10 @@ use crate::util::sync::AtomicFlag; use super::cache::NpmCache; -static NPM_REGISTRY_DEFAULT_URL: Lazy = Lazy::new(|| { - let env_var_name = "NPM_CONFIG_REGISTRY"; - if let Ok(registry_url) = std::env::var(env_var_name) { - // ensure there is a trailing slash for the directory - let registry_url = format!("{}/", registry_url.trim_end_matches('/')); - match Url::parse(®istry_url) { - Ok(url) => { - return url; - } - Err(err) => { - log::debug!("Invalid {} environment variable: {:#}", env_var_name, err,); - } - } - } - - Url::parse("https://registry.npmjs.org").unwrap() -}); - #[derive(Debug)] pub struct CliNpmRegistryApi(Option>); impl CliNpmRegistryApi { - pub fn default_url() -> &'static Url { - &NPM_REGISTRY_DEFAULT_URL - } - pub fn new( base_url: Url, cache: Arc, diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 05c1227a7c..f05275f3c4 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -34,7 +34,7 @@ use deno_semver::VersionReq; use crate::args::Lockfile; use crate::util::sync::TaskQueue; -use super::super::registry::CliNpmRegistryApi; +use super::CliNpmRegistryApi; /// Handles updating and storing npm resolution in memory where the underlying /// snapshot can be updated concurrently. Additionally handles updating the lockfile @@ -221,8 +221,6 @@ impl NpmResolution { .map(|pkg| pkg.id.clone()) } - // todo: NEXT - /// Resolves a package requirement for deno graph. This should only be /// called by deno_graph's NpmResolver or for resolving packages in /// a package.json @@ -275,14 +273,6 @@ impl NpmResolution { .all_system_packages_partitioned(system_info) } - // todo: NEXT - - pub fn has_packages(&self) -> bool { - !self.snapshot.read().is_empty() - } - - // todo: NEXT - pub fn snapshot(&self) -> NpmResolutionSnapshot { self.snapshot.read().clone() } @@ -293,8 +283,6 @@ impl NpmResolution { self.snapshot.read().as_valid_serialized() } - // todo: NEXT - pub fn serialized_valid_snapshot_for_system( &self, system_info: &NpmSystemInfo, diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 4076579bfc..b0f375779d 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -20,7 +20,7 @@ use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::NodePermissions; use deno_runtime::deno_node::NodeResolutionMode; -use crate::npm::NpmCache; +use super::super::cache::NpmCache; /// Part of the resolution that interacts with the file system. #[async_trait] diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 25db62f732..3f042a38b6 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -20,8 +20,7 @@ use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_node::NodePermissions; use deno_runtime::deno_node::NodeResolutionMode; -use crate::npm::NpmCache; - +use super::super::cache::NpmCache; use super::super::resolution::NpmResolution; use super::common::cache_packages; use super::common::types_package_name; diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 57170eccdf..8e4d72f269 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use std::sync::Arc; use crate::cache::CACHE_PERM; -use crate::npm::cache::mixed_case_package_name_decode; +use crate::npm::cache_dir::mixed_case_package_name_decode; use crate::util::fs::atomic_write_file; use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; use crate::util::fs::symlink_dir; @@ -41,11 +41,11 @@ use deno_semver::package::PackageNv; use serde::Deserialize; use serde::Serialize; -use crate::npm::cache::mixed_case_package_name_encode; -use crate::npm::NpmCache; +use crate::npm::cache_dir::mixed_case_package_name_encode; use crate::util::fs::copy_dir_recursive; use crate::util::fs::hard_link_dir_recursive; +use super::super::cache::NpmCache; use super::super::resolution::NpmResolution; use super::common::types_package_name; use super::common::NpmPackageFsResolver; diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index b6d96c4af3..5fc140f26c 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -11,14 +11,15 @@ use deno_core::url::Url; use deno_npm::NpmSystemInfo; use deno_runtime::deno_fs::FileSystem; -use crate::npm::NpmCache; use crate::util::progress_bar::ProgressBar; pub use self::common::NpmPackageFsResolver; + use self::global::GlobalNpmPackageResolver; use self::local::LocalNpmPackageResolver; -use super::NpmResolution; +use super::cache::NpmCache; +use super::resolution::NpmResolution; pub fn create_npm_fs_resolver( fs: Arc, diff --git a/cli/npm/tarball.rs b/cli/npm/managed/tarball.rs similarity index 100% rename from cli/npm/tarball.rs rename to cli/npm/managed/tarball.rs diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 114bf15f2d..22997a8b25 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -1,12 +1,8 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +mod cache_dir; mod managed; -// todo(#18967): move the cache, registry, and tarball into the managed folder -mod cache; -mod registry; -mod tarball; - use std::collections::HashMap; use std::path::PathBuf; use std::sync::Arc; @@ -17,20 +13,45 @@ use deno_core::url::Url; use deno_graph::NpmPackageReqResolution; use deno_npm::resolution::PackageReqNotFoundError; use deno_runtime::deno_node::NpmResolver; - -pub use cache::NpmCache; -pub use cache::NpmCacheDir; use deno_semver::npm::NpmPackageNvReference; use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; -pub use managed::create_npm_fs_resolver; -pub use managed::ManagedCliNpmResolver; -pub use managed::NpmPackageFsResolver; -pub use managed::NpmProcessState; -pub use managed::NpmResolution; -pub use managed::PackageJsonDepsInstaller; -pub use registry::CliNpmRegistryApi; + +pub use self::cache_dir::NpmCacheDir; +pub use self::managed::CliNpmResolverManagedCreateOptions; +pub use self::managed::CliNpmResolverManagedPackageJsonInstallerOption; +pub use self::managed::CliNpmResolverManagedSnapshotOption; +pub use self::managed::ManagedCliNpmResolver; + +pub enum CliNpmResolverCreateOptions { + Managed(CliNpmResolverManagedCreateOptions), + // todo(dsherret): implement this + #[allow(dead_code)] + Byonm, +} + +pub async fn create_cli_npm_resolver_for_lsp( + options: CliNpmResolverCreateOptions, +) -> Arc { + use CliNpmResolverCreateOptions::*; + match options { + Managed(options) => { + managed::create_managed_npm_resolver_for_lsp(options).await + } + Byonm => todo!(), + } +} + +pub async fn create_cli_npm_resolver( + options: CliNpmResolverCreateOptions, +) -> Result, AnyError> { + use CliNpmResolverCreateOptions::*; + match options { + Managed(options) => managed::create_managed_npm_resolver(options).await, + Byonm => todo!(), + } +} pub enum InnerCliNpmResolverRef<'a> { Managed(&'a ManagedCliNpmResolver), @@ -41,6 +62,8 @@ pub enum InnerCliNpmResolverRef<'a> { pub trait CliNpmResolver: NpmResolver { fn into_npm_resolver(self: Arc) -> Arc; + fn clone_snapshotted(&self) -> Arc; + fn root_dir_url(&self) -> &Url; fn as_inner(&self) -> InnerCliNpmResolverRef; diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index afe5a1b573..412cea12c1 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -36,11 +36,8 @@ use crate::args::PackageJsonDepsProvider; use crate::cache::DenoDir; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClient; -use crate::npm::CliNpmRegistryApi; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; -use crate::npm::NpmCache; -use crate::npm::NpmResolution; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -342,9 +339,6 @@ pub struct DenoCompileBinaryWriter<'a> { file_fetcher: &'a FileFetcher, client: &'a HttpClient, deno_dir: &'a DenoDir, - npm_api: &'a CliNpmRegistryApi, - npm_cache: &'a NpmCache, - npm_resolution: &'a NpmResolution, npm_resolver: &'a dyn CliNpmResolver, npm_system_info: NpmSystemInfo, package_json_deps_provider: &'a PackageJsonDepsProvider, @@ -356,9 +350,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { file_fetcher: &'a FileFetcher, client: &'a HttpClient, deno_dir: &'a DenoDir, - npm_api: &'a CliNpmRegistryApi, - npm_cache: &'a NpmCache, - npm_resolution: &'a NpmResolution, npm_resolver: &'a dyn CliNpmResolver, npm_system_info: NpmSystemInfo, package_json_deps_provider: &'a PackageJsonDepsProvider, @@ -367,11 +358,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { file_fetcher, client, deno_dir, - npm_api, - npm_cache, npm_resolver, npm_system_info, - npm_resolution, package_json_deps_provider, } } @@ -502,15 +490,22 @@ impl<'a> DenoCompileBinaryWriter<'a> { .resolve_import_map(self.file_fetcher) .await? .map(|import_map| (import_map.base_url().clone(), import_map.to_json())); - let (npm_vfs, npm_files) = if self.npm_resolution.has_packages() { - let (root_dir, files) = self.build_vfs()?.into_dir_and_files(); - let snapshot = self - .npm_resolution - .serialized_valid_snapshot_for_system(&self.npm_system_info); - eszip.add_npm_snapshot(snapshot); - (Some(root_dir), files) - } else { - (None, Vec::new()) + let (npm_vfs, npm_files) = match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + let (root_dir, files) = self.build_vfs()?.into_dir_and_files(); + eszip.add_npm_snapshot(snapshot); + (Some(root_dir), files) + } else { + (None, Vec::new()) + } + } + InnerCliNpmResolverRef::Byonm(_) => { + let (root_dir, files) = self.build_vfs()?.into_dir_and_files(); + (Some(root_dir), files) + } }; let metadata = Metadata { @@ -555,8 +550,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let registry_url = self.npm_api.base_url(); - let root_path = self.npm_cache.registry_folder(registry_url); + let registry_url = npm_resolver.registry_base_url(); + let root_path = + npm_resolver.registry_folder_in_global_cache(registry_url); let mut builder = VfsBuilder::new(root_path)?; for package in npm_resolver.all_system_packages(&self.npm_system_info) { diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 442334e398..64d56e6a51 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -14,14 +14,12 @@ use crate::http_util::HttpClient; use crate::module_loader::CjsResolutionStore; use crate::module_loader::NpmModuleLoader; use crate::node::CliCjsCodeAnalyzer; -use crate::npm::create_npm_fs_resolver; -use crate::npm::CliNpmRegistryApi; -use crate::npm::CliNpmResolver; -use crate::npm::ManagedCliNpmResolver; -use crate::npm::NpmCache; +use crate::npm::create_cli_npm_resolver; +use crate::npm::CliNpmResolverCreateOptions; +use crate::npm::CliNpmResolverManagedCreateOptions; +use crate::npm::CliNpmResolverManagedPackageJsonInstallerOption; +use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::NpmCacheDir; -use crate::npm::NpmResolution; -use crate::npm::PackageJsonDepsInstaller; use crate::resolver::MappedSpecifierResolver; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; @@ -40,7 +38,6 @@ use deno_core::ModuleLoader; use deno_core::ModuleSpecifier; use deno_core::ModuleType; use deno_core::ResolutionKind; -use deno_npm::NpmSystemInfo; use deno_runtime::deno_fs; use deno_runtime::deno_node::analyze::NodeCodeTranslator; use deno_runtime::deno_node::NodeResolver; @@ -309,82 +306,62 @@ pub async fn run( .join(format!("deno-compile-{}", current_exe_name)) .join("node_modules"); let npm_cache_dir = NpmCacheDir::new(root_path.clone()); - let (fs, vfs_root, node_modules_path, snapshot) = if let Some(snapshot) = - eszip.take_npm_snapshot() - { - let vfs_root_dir_path = if metadata.node_modules_dir { - root_path + let npm_global_cache_dir = npm_cache_dir.get_cache_location(); + let (fs, vfs_root, maybe_node_modules_path, maybe_snapshot) = + if let Some(snapshot) = eszip.take_npm_snapshot() { + let vfs_root_dir_path = if metadata.node_modules_dir { + root_path + } else { + npm_cache_dir.registry_folder(&npm_registry_url) + }; + let vfs = load_npm_vfs(vfs_root_dir_path.clone()) + .context("Failed to load npm vfs.")?; + let node_modules_path = if metadata.node_modules_dir { + Some(vfs.root().to_path_buf()) + } else { + None + }; + ( + Arc::new(DenoCompileFileSystem::new(vfs)) + as Arc, + Some(vfs_root_dir_path), + node_modules_path, + Some(snapshot), + ) } else { - npm_cache_dir.registry_folder(&npm_registry_url) + ( + Arc::new(deno_fs::RealFs) as Arc, + None, + None, + None, + ) }; - let vfs = load_npm_vfs(vfs_root_dir_path.clone()) - .context("Failed to load npm vfs.")?; - let node_modules_path = if metadata.node_modules_dir { - Some(vfs.root().to_path_buf()) - } else { - None - }; - ( - Arc::new(DenoCompileFileSystem::new(vfs)) as Arc, - Some(vfs_root_dir_path), - node_modules_path, - Some(snapshot), - ) - } else { - ( - Arc::new(deno_fs::RealFs) as Arc, - None, - None, - None, - ) - }; - let npm_cache = Arc::new(NpmCache::new( - npm_cache_dir, - CacheSetting::Only, - fs.clone(), - http_client.clone(), - progress_bar.clone(), - )); - let npm_api = Arc::new(CliNpmRegistryApi::new( - npm_registry_url.clone(), - npm_cache.clone(), - http_client.clone(), - progress_bar.clone(), - )); - let npm_resolution = Arc::new(NpmResolution::from_serialized( - npm_api.clone(), - snapshot, - None, - )); - let has_node_modules_dir = node_modules_path.is_some(); - let npm_fs_resolver = create_npm_fs_resolver( - fs.clone(), - npm_cache, - &progress_bar, - npm_registry_url, - npm_resolution.clone(), - node_modules_path, - NpmSystemInfo::default(), - ); + let has_node_modules_dir = maybe_node_modules_path.is_some(); let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new( metadata .package_json_deps .map(|serialized| serialized.into_deps()), )); - let package_json_installer = Arc::new(PackageJsonDepsInstaller::new( - package_json_deps_provider.clone(), - npm_api.clone(), - npm_resolution.clone(), - )); - let npm_resolver = Arc::new(ManagedCliNpmResolver::new( - npm_api.clone(), - fs.clone(), - npm_resolution.clone(), - npm_fs_resolver, - None, - package_json_installer, - )) as Arc; + let npm_resolver = create_cli_npm_resolver( + CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions { + snapshot: CliNpmResolverManagedSnapshotOption::Specified(maybe_snapshot), + maybe_lockfile: None, + fs: fs.clone(), + http_client: http_client.clone(), + npm_global_cache_dir, + cache_setting: CacheSetting::Only, + text_only_progress_bar: progress_bar, + maybe_node_modules_path, + package_json_installer: + CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall( + package_json_deps_provider.clone(), + ), + npm_registry_url, + npm_system_info: Default::default(), + }), + ) + .await?; let node_resolver = Arc::new(NodeResolver::new( fs.clone(), npm_resolver.clone().into_npm_resolver(), diff --git a/cli/tools/info.rs b/cli/tools/info.rs index e1972f08f5..fa0ede4371 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -98,7 +98,7 @@ fn print_cache_info( let dir = factory.deno_dir()?; #[allow(deprecated)] let modules_cache = factory.global_http_cache()?.get_global_cache_location(); - let npm_cache = factory.npm_cache()?.as_readonly().get_cache_location(); + let npm_cache = factory.deno_dir()?.npm_folder_path(); let typescript_cache = &dir.gen_cache.location; let registry_cache = dir.registries_folder_path(); let mut origin_dir = dir.origin_data_folder_path(); diff --git a/cli/tools/task.rs b/cli/tools/task.rs index d1513072a9..93e78dc568 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -86,11 +86,7 @@ pub async fn execute_script( // install the npm packages if we're using a managed resolver if let Some(npm_resolver) = npm_resolver.as_managed() { - let package_json_deps_installer = - factory.package_json_deps_installer().await?; - package_json_deps_installer - .ensure_top_level_install() - .await?; + npm_resolver.ensure_top_level_package_json_install().await?; npm_resolver.resolve_pending().await?; } diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs index fdea5fc26d..c324a56ddc 100644 --- a/cli/tools/vendor/mod.rs +++ b/cli/tools/vendor/mod.rs @@ -107,11 +107,12 @@ pub async fn vendor( .map(|config_path| config_path.parent().unwrap().join("node_modules")) }); if let Some(node_modules_path) = node_modules_path { - factory - .create_node_modules_npm_fs_resolver(node_modules_path) - .await? - .cache_packages() - .await?; + let cli_options = + cli_options.with_node_modules_dir_path(node_modules_path); + let factory = CliFactory::from_cli_options(Arc::new(cli_options)); + if let Some(managed) = factory.npm_resolver().await?.as_managed() { + managed.cache_packages().await?; + } } log::info!( concat!(