1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-25 16:49:18 -05:00

refactor: add NpmPackageId back from deno_graph as NpmPackageNodeId (#17804)

The `NpmPackageId` struct is being renamed to `NpmPackageNodeId`. In a
future PR it will be moved down into only npm dependency resolution and
a `NpmPackageId` struct will be introduced in `deno_graph` that only has
the name and version of the package (no peer dependency identifier
information). So a `NpmPackageReq` will map to an `NpmPackageId`, which
will map to an `NpmPackageNodeId` in the npm resolution.
This commit is contained in:
David Sherret 2023-02-17 09:12:22 -05:00 committed by GitHub
parent f8435d20b0
commit 610b8cc2bf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 494 additions and 243 deletions

1
Cargo.lock generated
View file

@ -860,6 +860,7 @@ dependencies = [
"test_util",
"text-size",
"text_lines",
"thiserror",
"tokio",
"tokio-util",
"tower-lsp",

View file

@ -99,6 +99,7 @@ shell-escape = "=0.1.5"
tar.workspace = true
text-size = "=1.1.0"
text_lines = "=0.6.0"
thiserror = "=1.0.38"
tokio.workspace = true
tokio-util.workspace = true
tower-lsp = { version = "=0.17.0", features = ["proposed"] }

View file

@ -12,6 +12,7 @@ pub use registry::NpmPackageVersionDistInfo;
pub use registry::NpmRegistryApi;
pub use registry::RealNpmRegistryApi;
pub use resolution::resolve_graph_npm_info;
pub use resolution::NpmPackageNodeId;
pub use resolution::NpmResolutionPackage;
pub use resolution::NpmResolutionSnapshot;
pub use resolvers::NpmPackageResolver;

File diff suppressed because it is too large Load diff

View file

@ -6,10 +6,11 @@ use std::collections::HashSet;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::parking_lot::RwLock;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_graph::semver::Version;
use serde::Deserialize;
use serde::Serialize;
use thiserror::Error;
use crate::args::Lockfile;
@ -30,9 +31,165 @@ use graph::Graph;
pub use snapshot::NpmResolutionSnapshot;
pub use specifier::resolve_graph_npm_info;
#[derive(Debug, Error)]
#[error("Invalid npm package id '{text}'. {message}")]
pub struct NpmPackageNodeIdDeserializationError {
message: String,
text: String,
}
/// A resolved unique identifier for an npm package. This contains
/// the resolved name, version, and peer dependency resolution identifiers.
#[derive(
Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize,
)]
pub struct NpmPackageNodeId {
pub name: String,
pub version: Version,
pub peer_dependencies: Vec<NpmPackageNodeId>,
}
impl NpmPackageNodeId {
#[allow(unused)]
pub fn scope(&self) -> Option<&str> {
if self.name.starts_with('@') && self.name.contains('/') {
self.name.split('/').next()
} else {
None
}
}
pub fn as_serialized(&self) -> String {
self.as_serialized_with_level(0)
}
fn as_serialized_with_level(&self, level: usize) -> String {
// WARNING: This should not change because it's used in the lockfile
let mut result = format!(
"{}@{}",
if level == 0 {
self.name.to_string()
} else {
self.name.replace('/', "+")
},
self.version
);
for peer in &self.peer_dependencies {
// unfortunately we can't do something like `_3` when
// this gets deep because npm package names can start
// with a number
result.push_str(&"_".repeat(level + 1));
result.push_str(&peer.as_serialized_with_level(level + 1));
}
result
}
pub fn from_serialized(
id: &str,
) -> Result<Self, NpmPackageNodeIdDeserializationError> {
use monch::*;
fn parse_name(input: &str) -> ParseResult<&str> {
if_not_empty(substring(move |input| {
for (pos, c) in input.char_indices() {
// first character might be a scope, so skip it
if pos > 0 && c == '@' {
return Ok((&input[pos..], ()));
}
}
ParseError::backtrace()
}))(input)
}
fn parse_version(input: &str) -> ParseResult<&str> {
if_not_empty(substring(skip_while(|c| c != '_')))(input)
}
fn parse_name_and_version(input: &str) -> ParseResult<(String, Version)> {
let (input, name) = parse_name(input)?;
let (input, _) = ch('@')(input)?;
let at_version_input = input;
let (input, version) = parse_version(input)?;
match Version::parse_from_npm(version) {
Ok(version) => Ok((input, (name.to_string(), version))),
Err(err) => ParseError::fail(at_version_input, format!("{err:#}")),
}
}
fn parse_level_at_level<'a>(
level: usize,
) -> impl Fn(&'a str) -> ParseResult<'a, ()> {
fn parse_level(input: &str) -> ParseResult<usize> {
let level = input.chars().take_while(|c| *c == '_').count();
Ok((&input[level..], level))
}
move |input| {
let (input, parsed_level) = parse_level(input)?;
if parsed_level == level {
Ok((input, ()))
} else {
ParseError::backtrace()
}
}
}
fn parse_peers_at_level<'a>(
level: usize,
) -> impl Fn(&'a str) -> ParseResult<'a, Vec<NpmPackageNodeId>> {
move |mut input| {
let mut peers = Vec::new();
while let Ok((level_input, _)) = parse_level_at_level(level)(input) {
input = level_input;
let peer_result = parse_id_at_level(level)(input)?;
input = peer_result.0;
peers.push(peer_result.1);
}
Ok((input, peers))
}
}
fn parse_id_at_level<'a>(
level: usize,
) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageNodeId> {
move |input| {
let (input, (name, version)) = parse_name_and_version(input)?;
let name = if level > 0 {
name.replace('+', "/")
} else {
name
};
let (input, peer_dependencies) =
parse_peers_at_level(level + 1)(input)?;
Ok((
input,
NpmPackageNodeId {
name,
version,
peer_dependencies,
},
))
}
}
with_failure_handling(parse_id_at_level(0))(id).map_err(|err| {
NpmPackageNodeIdDeserializationError {
message: format!("{err:#}"),
text: id.to_string(),
}
})
}
pub fn display(&self) -> String {
// Don't implement std::fmt::Display because we don't
// want this to be used by accident in certain scenarios.
format!("{}@{}", self.name, self.version)
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct NpmResolutionPackage {
pub id: NpmPackageId,
pub id: NpmPackageNodeId,
/// The peer dependency resolution can differ for the same
/// package (name and version) depending on where it is in
/// the resolution tree. This copy index indicates which
@ -41,7 +198,7 @@ pub struct NpmResolutionPackage {
pub dist: NpmPackageVersionDistInfo,
/// Key is what the package refers to the other package as,
/// which could be different from the package name.
pub dependencies: HashMap<String, NpmPackageId>,
pub dependencies: HashMap<String, NpmPackageNodeId>,
}
impl NpmResolutionPackage {
@ -189,14 +346,14 @@ impl NpmResolution {
pub fn resolve_package_from_id(
&self,
id: &NpmPackageId,
id: &NpmPackageNodeId,
) -> Option<NpmResolutionPackage> {
self.snapshot.read().package_from_id(id).cloned()
}
pub fn resolve_package_cache_folder_id_from_id(
&self,
id: &NpmPackageId,
id: &NpmPackageNodeId,
) -> Option<NpmPackageCacheFolderId> {
self
.snapshot
@ -255,3 +412,50 @@ impl NpmResolution {
Ok(())
}
}
#[cfg(test)]
mod test {
use deno_graph::semver::Version;
use super::NpmPackageNodeId;
#[test]
fn serialize_npm_package_id() {
let id = NpmPackageNodeId {
name: "pkg-a".to_string(),
version: Version::parse_from_npm("1.2.3").unwrap(),
peer_dependencies: vec![
NpmPackageNodeId {
name: "pkg-b".to_string(),
version: Version::parse_from_npm("3.2.1").unwrap(),
peer_dependencies: vec![
NpmPackageNodeId {
name: "pkg-c".to_string(),
version: Version::parse_from_npm("1.3.2").unwrap(),
peer_dependencies: vec![],
},
NpmPackageNodeId {
name: "pkg-d".to_string(),
version: Version::parse_from_npm("2.3.4").unwrap(),
peer_dependencies: vec![],
},
],
},
NpmPackageNodeId {
name: "pkg-e".to_string(),
version: Version::parse_from_npm("2.3.1").unwrap(),
peer_dependencies: vec![NpmPackageNodeId {
name: "pkg-f".to_string(),
version: Version::parse_from_npm("2.3.1").unwrap(),
peer_dependencies: vec![],
}],
},
],
};
// this shouldn't change because it's used in the lockfile
let serialized = id.as_serialized();
assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1");
assert_eq!(NpmPackageNodeId::from_serialized(&serialized).unwrap(), id);
}
}

View file

@ -10,7 +10,6 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::parking_lot::Mutex;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_graph::semver::VersionReq;
use serde::Deserialize;
@ -23,6 +22,7 @@ use crate::npm::registry::NpmPackageVersionDistInfo;
use crate::npm::registry::NpmRegistryApi;
use crate::npm::registry::RealNpmRegistryApi;
use super::NpmPackageNodeId;
use super::NpmResolutionPackage;
/// Packages partitioned by if they are "copy" packages or not.
@ -47,10 +47,10 @@ impl NpmPackagesPartitioned {
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct NpmResolutionSnapshot {
#[serde(with = "map_to_vec")]
pub(super) package_reqs: HashMap<NpmPackageReq, NpmPackageId>,
pub(super) packages_by_name: HashMap<String, Vec<NpmPackageId>>,
pub(super) package_reqs: HashMap<NpmPackageReq, NpmPackageNodeId>,
pub(super) packages_by_name: HashMap<String, Vec<NpmPackageNodeId>>,
#[serde(with = "map_to_vec")]
pub(super) packages: HashMap<NpmPackageId, NpmResolutionPackage>,
pub(super) packages: HashMap<NpmPackageNodeId, NpmResolutionPackage>,
}
// This is done so the maps with non-string keys get serialized and deserialized as vectors.
@ -104,7 +104,7 @@ impl NpmResolutionSnapshot {
}
}
pub fn top_level_packages(&self) -> Vec<NpmPackageId> {
pub fn top_level_packages(&self) -> Vec<NpmPackageNodeId> {
self
.package_reqs
.values()
@ -116,7 +116,7 @@ impl NpmResolutionSnapshot {
pub fn package_from_id(
&self,
id: &NpmPackageId,
id: &NpmPackageNodeId,
) -> Option<&NpmResolutionPackage> {
self.packages.get(id)
}
@ -198,10 +198,10 @@ impl NpmResolutionSnapshot {
&self,
name: &str,
version_req: &VersionReq,
) -> Option<NpmPackageId> {
) -> Option<NpmPackageNodeId> {
// todo(dsherret): this is not exactly correct because some ids
// will be better than others due to peer dependencies
let mut maybe_best_id: Option<&NpmPackageId> = None;
let mut maybe_best_id: Option<&NpmPackageNodeId> = None;
if let Some(ids) = self.packages_by_name.get(name) {
for id in ids {
if version_req.matches(&id.version) {
@ -222,9 +222,9 @@ impl NpmResolutionSnapshot {
lockfile: Arc<Mutex<Lockfile>>,
api: &RealNpmRegistryApi,
) -> Result<Self, AnyError> {
let mut package_reqs: HashMap<NpmPackageReq, NpmPackageId>;
let mut packages_by_name: HashMap<String, Vec<NpmPackageId>>;
let mut packages: HashMap<NpmPackageId, NpmResolutionPackage>;
let mut package_reqs: HashMap<NpmPackageReq, NpmPackageNodeId>;
let mut packages_by_name: HashMap<String, Vec<NpmPackageNodeId>>;
let mut packages: HashMap<NpmPackageNodeId, NpmResolutionPackage>;
let mut copy_index_resolver: SnapshotPackageCopyIndexResolver;
{
@ -244,14 +244,14 @@ impl NpmResolutionSnapshot {
for (key, value) in &lockfile.content.npm.specifiers {
let package_req = NpmPackageReq::from_str(key)
.with_context(|| format!("Unable to parse npm specifier: {key}"))?;
let package_id = NpmPackageId::from_serialized(value)?;
let package_id = NpmPackageNodeId::from_serialized(value)?;
package_reqs.insert(package_req, package_id.clone());
verify_ids.insert(package_id.clone());
}
// then the packages
for (key, value) in &lockfile.content.npm.packages {
let package_id = NpmPackageId::from_serialized(key)?;
let package_id = NpmPackageNodeId::from_serialized(key)?;
// collect the dependencies
let mut dependencies = HashMap::default();
@ -262,7 +262,7 @@ impl NpmResolutionSnapshot {
.push(package_id.clone());
for (name, specifier) in &value.dependencies {
let dep_id = NpmPackageId::from_serialized(specifier)?;
let dep_id = NpmPackageNodeId::from_serialized(specifier)?;
dependencies.insert(name.to_string(), dep_id.clone());
verify_ids.insert(dep_id);
}
@ -336,7 +336,7 @@ impl NpmResolutionSnapshot {
}
pub struct SnapshotPackageCopyIndexResolver {
packages_to_copy_index: HashMap<NpmPackageId, usize>,
packages_to_copy_index: HashMap<NpmPackageNodeId, usize>,
package_name_version_to_copy_count: HashMap<(String, String), usize>,
}
@ -349,7 +349,7 @@ impl SnapshotPackageCopyIndexResolver {
}
pub fn from_map_with_capacity(
mut packages_to_copy_index: HashMap<NpmPackageId, usize>,
mut packages_to_copy_index: HashMap<NpmPackageNodeId, usize>,
capacity: usize,
) -> Self {
let mut package_name_version_to_copy_count =
@ -372,7 +372,7 @@ impl SnapshotPackageCopyIndexResolver {
}
}
pub fn resolve(&mut self, id: &NpmPackageId) -> usize {
pub fn resolve(&mut self, id: &NpmPackageNodeId) -> usize {
if let Some(index) = self.packages_to_copy_index.get(id) {
*index
} else {
@ -422,24 +422,24 @@ mod tests {
SnapshotPackageCopyIndexResolver::with_capacity(10);
assert_eq!(
copy_index_resolver
.resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()),
.resolve(&NpmPackageNodeId::from_serialized("package@1.0.0").unwrap()),
0
);
assert_eq!(
copy_index_resolver
.resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()),
.resolve(&NpmPackageNodeId::from_serialized("package@1.0.0").unwrap()),
0
);
assert_eq!(
copy_index_resolver.resolve(
&NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0")
&NpmPackageNodeId::from_serialized("package@1.0.0_package-b@1.0.0")
.unwrap()
),
1
);
assert_eq!(
copy_index_resolver.resolve(
&NpmPackageId::from_serialized(
&NpmPackageNodeId::from_serialized(
"package@1.0.0_package-b@1.0.0__package-c@2.0.0"
)
.unwrap()
@ -448,14 +448,15 @@ mod tests {
);
assert_eq!(
copy_index_resolver.resolve(
&NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0")
&NpmPackageNodeId::from_serialized("package@1.0.0_package-b@1.0.0")
.unwrap()
),
1
);
assert_eq!(
copy_index_resolver
.resolve(&NpmPackageId::from_serialized("package-b@1.0.0").unwrap()),
copy_index_resolver.resolve(
&NpmPackageNodeId::from_serialized("package-b@1.0.0").unwrap()
),
0
);
}

View file

@ -10,7 +10,6 @@ use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::future::BoxFuture;
use deno_core::url::Url;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -19,6 +18,7 @@ use crate::args::Lockfile;
use crate::npm::cache::should_sync_download;
use crate::npm::resolution::NpmResolutionSnapshot;
use crate::npm::NpmCache;
use crate::npm::NpmPackageNodeId;
use crate::npm::NpmResolutionPackage;
pub trait InnerNpmPackageResolver: Send + Sync {
@ -39,7 +39,10 @@ pub trait InnerNpmPackageResolver: Send + Sync {
specifier: &ModuleSpecifier,
) -> Result<PathBuf, AnyError>;
fn package_size(&self, package_id: &NpmPackageId) -> Result<u64, AnyError>;
fn package_size(
&self,
package_id: &NpmPackageNodeId,
) -> Result<u64, AnyError>;
fn has_packages(&self) -> bool;

View file

@ -12,7 +12,6 @@ use deno_core::error::AnyError;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::FutureExt;
use deno_core::url::Url;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -23,6 +22,7 @@ use crate::npm::resolution::NpmResolution;
use crate::npm::resolution::NpmResolutionSnapshot;
use crate::npm::resolvers::common::cache_packages;
use crate::npm::NpmCache;
use crate::npm::NpmPackageNodeId;
use crate::npm::NpmResolutionPackage;
use crate::npm::RealNpmRegistryApi;
@ -54,7 +54,7 @@ impl GlobalNpmPackageResolver {
}
}
fn package_folder(&self, id: &NpmPackageId) -> PathBuf {
fn package_folder(&self, id: &NpmPackageNodeId) -> PathBuf {
let folder_id = self
.resolution
.resolve_package_cache_folder_id_from_id(id)
@ -125,7 +125,10 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
)
}
fn package_size(&self, package_id: &NpmPackageId) -> Result<u64, AnyError> {
fn package_size(
&self,
package_id: &NpmPackageNodeId,
) -> Result<u64, AnyError> {
let package_folder = self.package_folder(package_id);
Ok(crate::util::fs::dir_size(&package_folder)?)
}

View file

@ -18,7 +18,6 @@ use deno_core::error::AnyError;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::FutureExt;
use deno_core::url::Url;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_runtime::deno_core::futures;
use deno_runtime::deno_node::NodePermissions;
@ -33,6 +32,7 @@ use crate::npm::cache::NpmPackageCacheFolderId;
use crate::npm::resolution::NpmResolution;
use crate::npm::resolution::NpmResolutionSnapshot;
use crate::npm::NpmCache;
use crate::npm::NpmPackageNodeId;
use crate::npm::NpmResolutionPackage;
use crate::npm::RealNpmRegistryApi;
use crate::util::fs::copy_dir_recursive;
@ -112,7 +112,7 @@ impl LocalNpmPackageResolver {
fn get_package_id_folder(
&self,
package_id: &NpmPackageId,
package_id: &NpmPackageNodeId,
) -> Result<PathBuf, AnyError> {
match self.resolution.resolve_package_from_id(package_id) {
Some(package) => Ok(self.get_package_id_folder_from_package(&package)),
@ -203,7 +203,10 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver {
Ok(package_root_path)
}
fn package_size(&self, package_id: &NpmPackageId) -> Result<u64, AnyError> {
fn package_size(
&self,
package_id: &NpmPackageNodeId,
) -> Result<u64, AnyError> {
let package_folder_path = self.get_package_id_folder(package_id)?;
Ok(crate::util::fs::dir_size(&package_folder_path)?)

View file

@ -11,7 +11,6 @@ use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::serde_json;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReq;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::deno_node::NodeResolutionMode;
@ -32,6 +31,7 @@ use crate::util::fs::canonicalize_path_maybe_not_exists;
use self::common::InnerNpmPackageResolver;
use self::local::LocalNpmPackageResolver;
use super::NpmCache;
use super::NpmPackageNodeId;
use super::NpmResolutionSnapshot;
use super::RealNpmRegistryApi;
@ -225,7 +225,7 @@ impl NpmPackageResolver {
/// Attempts to get the package size in bytes.
pub fn package_size(
&self,
package_id: &NpmPackageId,
package_id: &NpmPackageNodeId,
) -> Result<u64, AnyError> {
self.inner.package_size(package_id)
}

View file

@ -10,7 +10,6 @@ use deno_core::error::AnyError;
use deno_core::resolve_url_or_path;
use deno_core::serde_json;
use deno_core::serde_json::json;
use deno_graph::npm::NpmPackageId;
use deno_graph::npm::NpmPackageReference;
use deno_graph::npm::NpmPackageReq;
use deno_graph::Dependency;
@ -23,6 +22,7 @@ use deno_runtime::colors;
use crate::args::Flags;
use crate::args::InfoFlags;
use crate::display;
use crate::npm::NpmPackageNodeId;
use crate::npm::NpmPackageResolver;
use crate::npm::NpmResolutionPackage;
use crate::npm::NpmResolutionSnapshot;
@ -297,9 +297,9 @@ fn print_tree_node<TWrite: Write>(
/// Precached information about npm packages that are used in deno info.
#[derive(Default)]
struct NpmInfo {
package_sizes: HashMap<NpmPackageId, u64>,
resolved_reqs: HashMap<NpmPackageReq, NpmPackageId>,
packages: HashMap<NpmPackageId, NpmResolutionPackage>,
package_sizes: HashMap<NpmPackageNodeId, u64>,
resolved_reqs: HashMap<NpmPackageReq, NpmPackageNodeId>,
packages: HashMap<NpmPackageNodeId, NpmResolutionPackage>,
specifiers: HashMap<ModuleSpecifier, NpmPackageReq>,
}