// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures; use deno_core::parking_lot::RwLock; use serde::Deserialize; use serde::Serialize; use super::registry::NpmPackageInfo; use super::registry::NpmPackageVersionDistInfo; use super::registry::NpmPackageVersionInfo; use super::registry::NpmRegistryApi; use super::semver::NpmVersion; use super::semver::SpecifierVersionReq; /// The version matcher used for npm schemed urls is more strict than /// the one used by npm packages and so we represent either via a trait. pub trait NpmVersionMatcher { fn matches(&self, version: &NpmVersion) -> bool; fn is_latest(&self) -> bool; fn version_text(&self) -> String; } #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct NpmPackageReference { pub req: NpmPackageReq, pub sub_path: Option, } impl NpmPackageReference { pub fn from_specifier( specifier: &ModuleSpecifier, ) -> Result { Self::from_str(specifier.as_str()) } pub fn from_str(specifier: &str) -> Result { let specifier = match specifier.strip_prefix("npm:") { Some(s) => s, None => { bail!("Not an npm specifier: {}", specifier); } }; let parts = specifier.split('/').collect::>(); let name_part_len = if specifier.starts_with('@') { 2 } else { 1 }; if parts.len() < name_part_len { return Err(generic_error(format!("Not a valid package: {}", specifier))); } let name_parts = &parts[0..name_part_len]; let last_name_part = &name_parts[name_part_len - 1]; let (name, version_req) = if let Some(at_index) = last_name_part.rfind('@') { let version = &last_name_part[at_index + 1..]; let last_name_part = &last_name_part[..at_index]; let version_req = SpecifierVersionReq::parse(version) .with_context(|| "Invalid version requirement.")?; let name = if name_part_len == 1 { last_name_part.to_string() } else { format!("{}/{}", name_parts[0], last_name_part) }; (name, Some(version_req)) } else { (name_parts.join("/"), None) }; let sub_path = if parts.len() == name_parts.len() { None } else { Some(parts[name_part_len..].join("/")) }; if let Some(sub_path) = &sub_path { if let Some(at_index) = sub_path.rfind('@') { let (new_sub_path, version) = sub_path.split_at(at_index); let msg = format!( "Invalid package specifier 'npm:{}/{}'. Did you mean to write 'npm:{}{}/{}'?", name, sub_path, name, version, new_sub_path ); return Err(generic_error(msg)); } } Ok(NpmPackageReference { req: NpmPackageReq { name, version_req }, sub_path, }) } } impl std::fmt::Display for NpmPackageReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if let Some(sub_path) = &self.sub_path { write!(f, "{}/{}", self.req, sub_path) } else { write!(f, "{}", self.req) } } } #[derive( Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize, )] pub struct NpmPackageReq { pub name: String, pub version_req: Option, } impl std::fmt::Display for NpmPackageReq { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match &self.version_req { Some(req) => write!(f, "{}@{}", self.name, req), None => write!(f, "{}", self.name), } } } impl NpmVersionMatcher for NpmPackageReq { fn matches(&self, version: &NpmVersion) -> bool { match &self.version_req { Some(req) => req.matches(version), None => version.pre.is_empty(), } } fn is_latest(&self) -> bool { self.version_req.is_none() } fn version_text(&self) -> String { self .version_req .as_ref() .map(|v| format!("{}", v)) .unwrap_or_else(|| "non-prerelease".to_string()) } } #[derive( Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, )] pub struct NpmPackageId { pub name: String, pub version: NpmVersion, } impl NpmPackageId { #[allow(unused)] pub fn scope(&self) -> Option<&str> { if self.name.starts_with('@') && self.name.contains('/') { self.name.split('/').next() } else { None } } } impl std::fmt::Display for NpmPackageId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}@{}", self.name, self.version) } } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NpmResolutionPackage { pub id: NpmPackageId, pub dist: NpmPackageVersionDistInfo, /// Key is what the package refers to the other package as, /// which could be different from the package name. pub dependencies: HashMap, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct NpmResolutionSnapshot { #[serde(with = "map_to_vec")] package_reqs: HashMap, packages_by_name: HashMap>, #[serde(with = "map_to_vec")] packages: HashMap, } // This is done so the maps with non-string keys get serialized and deserialized as vectors. // Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792 mod map_to_vec { use std::collections::HashMap; use serde::de::Deserialize; use serde::de::Deserializer; use serde::ser::Serializer; use serde::Serialize; pub fn serialize( map: &HashMap, serializer: S, ) -> Result where S: Serializer, { serializer.collect_seq(map.iter()) } pub fn deserialize< 'de, D, K: Deserialize<'de> + Eq + std::hash::Hash, V: Deserialize<'de>, >( deserializer: D, ) -> Result, D::Error> where D: Deserializer<'de>, { let mut map = HashMap::new(); for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? { map.insert(key, value); } Ok(map) } } impl NpmResolutionSnapshot { /// Resolve a node package from a deno module. pub fn resolve_package_from_deno_module( &self, req: &NpmPackageReq, ) -> Result<&NpmResolutionPackage, AnyError> { match self.package_reqs.get(req) { Some(version) => Ok( self .packages .get(&NpmPackageId { name: req.name.clone(), version: version.clone(), }) .unwrap(), ), None => bail!("could not find npm package directory for '{}'", req), } } pub fn top_level_packages(&self) -> Vec { self .package_reqs .iter() .map(|(req, version)| NpmPackageId { name: req.name.clone(), version: version.clone(), }) .collect::>() .into_iter() .collect::>() } pub fn package_from_id( &self, id: &NpmPackageId, ) -> Option<&NpmResolutionPackage> { self.packages.get(id) } pub fn resolve_package_from_package( &self, name: &str, referrer: &NpmPackageId, ) -> Result<&NpmResolutionPackage, AnyError> { match self.packages.get(referrer) { Some(referrer_package) => { let name_ = name_without_path(name); if let Some(id) = referrer_package.dependencies.get(name_) { return Ok(self.packages.get(id).unwrap()); } if referrer_package.id.name == name_ { return Ok(referrer_package); } // TODO(bartlomieju): this should use a reverse lookup table in the // snapshot instead of resolving best version again. let req = NpmPackageReq { name: name_.to_string(), version_req: None, }; if let Some(version) = self.resolve_best_package_version(name_, &req) { let id = NpmPackageId { name: name_.to_string(), version, }; if let Some(pkg) = self.packages.get(&id) { return Ok(pkg); } } bail!( "could not find npm package '{}' referenced by '{}'", name, referrer ) } None => bail!("could not find referrer npm package '{}'", referrer), } } pub fn all_packages(&self) -> Vec { self.packages.values().cloned().collect() } pub fn resolve_best_package_version( &self, name: &str, version_matcher: &impl NpmVersionMatcher, ) -> Option { let mut maybe_best_version: Option<&NpmVersion> = None; if let Some(versions) = self.packages_by_name.get(name) { for version in versions { if version_matcher.matches(version) { let is_best_version = maybe_best_version .as_ref() .map(|best_version| (*best_version).cmp(version).is_lt()) .unwrap_or(true); if is_best_version { maybe_best_version = Some(version); } } } } maybe_best_version.cloned() } } pub struct NpmResolution { api: NpmRegistryApi, snapshot: RwLock, update_sempahore: tokio::sync::Semaphore, } impl std::fmt::Debug for NpmResolution { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snapshot = self.snapshot.read(); f.debug_struct("NpmResolution") .field("snapshot", &snapshot) .finish() } } impl NpmResolution { pub fn new( api: NpmRegistryApi, initial_snapshot: Option, ) -> Self { Self { api, snapshot: RwLock::new(initial_snapshot.unwrap_or_default()), update_sempahore: tokio::sync::Semaphore::new(1), } } pub async fn add_package_reqs( &self, mut packages: Vec, ) -> Result<(), AnyError> { // multiple packages are resolved in alphabetical order packages.sort_by(|a, b| a.name.cmp(&b.name)); // only allow one thread in here at a time let _permit = self.update_sempahore.acquire().await.unwrap(); let mut snapshot = self.snapshot.read().clone(); let mut pending_dependencies = VecDeque::new(); // go over the top level packages first, then down the // tree one level at a time through all the branches for package_ref in packages { if snapshot.package_reqs.contains_key(&package_ref) { // skip analyzing this package, as there's already a matching top level package continue; } // inspect the list of current packages if let Some(version) = snapshot.resolve_best_package_version(&package_ref.name, &package_ref) { snapshot.package_reqs.insert(package_ref, version); continue; // done, no need to continue } // no existing best version, so resolve the current packages let info = self.api.package_info(&package_ref.name).await?; let version_and_info = get_resolved_package_version_and_info( &package_ref.name, &package_ref, info, None, )?; let id = NpmPackageId { name: package_ref.name.clone(), version: version_and_info.version.clone(), }; let dependencies = version_and_info .info .dependencies_as_entries() .with_context(|| format!("npm package: {}", id))?; pending_dependencies.push_back((id.clone(), dependencies)); snapshot.packages.insert( id.clone(), NpmResolutionPackage { id, dist: version_and_info.info.dist, dependencies: Default::default(), }, ); snapshot .packages_by_name .entry(package_ref.name.clone()) .or_default() .push(version_and_info.version.clone()); snapshot .package_reqs .insert(package_ref, version_and_info.version); } // now go down through the dependencies by tree depth while let Some((parent_package_id, mut deps)) = pending_dependencies.pop_front() { // sort the dependencies alphabetically by name then by version descending deps.sort_by(|a, b| match a.name.cmp(&b.name) { // sort by newest to oldest Ordering::Equal => b .version_req .version_text() .cmp(&a.version_req.version_text()), ordering => ordering, }); // cache all the dependencies' registry infos in parallel when this env var isn't set if std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") != Ok("1".to_string()) { let handles = deps .iter() .map(|dep| { let name = dep.name.clone(); let api = self.api.clone(); tokio::task::spawn(async move { // it's ok to call this without storing the result, because // NpmRegistryApi will cache the package info in memory api.package_info(&name).await }) }) .collect::>(); let results = futures::future::join_all(handles).await; for result in results { result??; // surface the first error } } // now resolve them for dep in deps { // check if an existing dependency matches this let id = if let Some(version) = snapshot.resolve_best_package_version(&dep.name, &dep.version_req) { NpmPackageId { name: dep.name.clone(), version, } } else { // get the information let info = self.api.package_info(&dep.name).await?; let version_and_info = get_resolved_package_version_and_info( &dep.name, &dep.version_req, info, None, )?; let dependencies = version_and_info .info .dependencies_as_entries() .with_context(|| { format!("npm package: {}@{}", dep.name, version_and_info.version) })?; let id = NpmPackageId { name: dep.name.clone(), version: version_and_info.version.clone(), }; pending_dependencies.push_back((id.clone(), dependencies)); snapshot.packages.insert( id.clone(), NpmResolutionPackage { id: id.clone(), dist: version_and_info.info.dist, dependencies: Default::default(), }, ); snapshot .packages_by_name .entry(dep.name.clone()) .or_default() .push(id.version.clone()); id }; // add this version as a dependency of the package snapshot .packages .get_mut(&parent_package_id) .unwrap() .dependencies .insert(dep.bare_specifier.clone(), id); } } *self.snapshot.write() = snapshot; Ok(()) } pub fn resolve_package_from_package( &self, name: &str, referrer: &NpmPackageId, ) -> Result { self .snapshot .read() .resolve_package_from_package(name, referrer) .cloned() } /// Resolve a node package from a deno module. pub fn resolve_package_from_deno_module( &self, package: &NpmPackageReq, ) -> Result { self .snapshot .read() .resolve_package_from_deno_module(package) .cloned() } pub fn all_packages(&self) -> Vec { self.snapshot.read().all_packages() } pub fn has_packages(&self) -> bool { !self.snapshot.read().packages.is_empty() } pub fn snapshot(&self) -> NpmResolutionSnapshot { self.snapshot.read().clone() } } #[derive(Clone)] struct VersionAndInfo { version: NpmVersion, info: NpmPackageVersionInfo, } fn get_resolved_package_version_and_info( pkg_name: &str, version_matcher: &impl NpmVersionMatcher, info: NpmPackageInfo, parent: Option<&NpmPackageId>, ) -> Result { let mut maybe_best_version: Option = None; if version_matcher.is_latest() { if let Some(version) = info.dist_tags.get("latest") { match info.versions.get(version) { Some(info) => { return Ok(VersionAndInfo { version: NpmVersion::parse(version)?, info: info.clone(), }); } None => { bail!( "Could not find version '{}' referenced in dist-tag 'latest'.", version, ) } } } } for (_, version_info) in info.versions.into_iter() { let version = NpmVersion::parse(&version_info.version)?; if version_matcher.matches(&version) { let is_best_version = maybe_best_version .as_ref() .map(|best_version| best_version.version.cmp(&version).is_lt()) .unwrap_or(true); if is_best_version { maybe_best_version = Some(VersionAndInfo { version, info: version_info, }); } } } match maybe_best_version { Some(v) => Ok(v), // If the package isn't found, it likely means that the user needs to use // `--reload` to get the latest npm package information. Although it seems // like we could make this smart by fetching the latest information for // this package here, we really need a full restart. There could be very // interesting bugs that occur if this package's version was resolved by // something previous using the old information, then now being smart here // causes a new fetch of the package information, meaning this time the // previous resolution of this package's version resolved to an older // version, but next time to a different version because it has new information. None => bail!( concat!( "Could not find npm package '{}' matching {}{}. ", "Try retrieving the latest npm package information by running with --reload", ), pkg_name, version_matcher.version_text(), match parent { Some(id) => format!(" as specified in {}", id), None => String::new(), } ), } } fn name_without_path(name: &str) -> &str { let mut search_start_index = 0; if name.starts_with('@') { if let Some(slash_index) = name.find('/') { search_start_index = slash_index + 1; } } if let Some(slash_index) = &name[search_start_index..].find('/') { // get the name up until the path slash &name[0..search_start_index + slash_index] } else { name } } #[cfg(test)] mod tests { use super::*; #[test] fn parse_npm_package_ref() { assert_eq!( NpmPackageReference::from_str("npm:@package/test").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "@package/test".to_string(), version_req: None, }, sub_path: None, } ); assert_eq!( NpmPackageReference::from_str("npm:@package/test@1").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "@package/test".to_string(), version_req: Some(SpecifierVersionReq::parse("1").unwrap()), }, sub_path: None, } ); assert_eq!( NpmPackageReference::from_str("npm:@package/test@~1.1/sub_path").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "@package/test".to_string(), version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), }, sub_path: Some("sub_path".to_string()), } ); assert_eq!( NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "@package/test".to_string(), version_req: None, }, sub_path: Some("sub_path".to_string()), } ); assert_eq!( NpmPackageReference::from_str("npm:test").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "test".to_string(), version_req: None, }, sub_path: None, } ); assert_eq!( NpmPackageReference::from_str("npm:test@^1.2").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "test".to_string(), version_req: Some(SpecifierVersionReq::parse("^1.2").unwrap()), }, sub_path: None, } ); assert_eq!( NpmPackageReference::from_str("npm:test@~1.1/sub_path").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "test".to_string(), version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), }, sub_path: Some("sub_path".to_string()), } ); assert_eq!( NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), NpmPackageReference { req: NpmPackageReq { name: "@package/test".to_string(), version_req: None, }, sub_path: Some("sub_path".to_string()), } ); assert_eq!( NpmPackageReference::from_str("npm:@package") .err() .unwrap() .to_string(), "Not a valid package: @package" ); } #[test] fn test_name_without_path() { assert_eq!(name_without_path("foo"), "foo"); assert_eq!(name_without_path("@foo/bar"), "@foo/bar"); assert_eq!(name_without_path("@foo/bar/baz"), "@foo/bar"); assert_eq!(name_without_path("@hello"), "@hello"); } #[test] fn test_get_resolved_package_version_and_info() { // dist tag where version doesn't exist let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); let result = get_resolved_package_version_and_info( "test", &package_ref.req, NpmPackageInfo { name: "test".to_string(), versions: HashMap::new(), dist_tags: HashMap::from([( "latest".to_string(), "1.0.0-alpha".to_string(), )]), }, None, ); assert_eq!( result.err().unwrap().to_string(), "Could not find version '1.0.0-alpha' referenced in dist-tag 'latest'." ); // dist tag where version is a pre-release let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); let result = get_resolved_package_version_and_info( "test", &package_ref.req, NpmPackageInfo { name: "test".to_string(), versions: HashMap::from([ ("0.1.0".to_string(), NpmPackageVersionInfo::default()), ( "1.0.0-alpha".to_string(), NpmPackageVersionInfo { version: "0.1.0-alpha".to_string(), ..Default::default() }, ), ]), dist_tags: HashMap::from([( "latest".to_string(), "1.0.0-alpha".to_string(), )]), }, None, ); assert_eq!(result.unwrap().version.to_string(), "1.0.0-alpha"); } }