foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00
Code Issues Packages 1 Wiki Activity

fix(byonm): resolve npm deps of jsr deps (#25399)

Browse source 
This allows using npm deps of jsr deps without having to add them to the
root package.json.

Works by taking the package requirement and scanning the
`node_modules/.deno` directory for the best matching package, so it
relies on deno's node_modules structure.

Additionally to make the transition from package.json to deno.json
easier, Deno now:

1. Installs npm deps in a deno.json at the same time as installing npm
deps from a package.json.
2. Uses the alias in the import map for `node_modules/<alias>` for
better package.json compatiblity.
This commit is contained in:
David Sherret 2024-09-04 16:00:44 +02:00 committed by GitHub
parent 13911eb8ef
commit c6d1b0a1cc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 504 additions and 145 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cli/args/mod.rs
View file

@ -44,7 +44,7 @@ pub use deno_config::deno_json::TsTypeLib;
pub use deno_config::glob::FilePatterns;
pub use flags::*;
pub use lockfile::CliLockfile;
pub use package_json::PackageJsonInstallDepsProvider;
pub use package_json::NpmInstallDepsProvider;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;

129
cli/args/package_json.rs
View file

@ -1,17 +1,20 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::path::PathBuf;
use std::sync::Arc;
use deno_config::workspace::Workspace;
use deno_core::serde_json;
use deno_package_json::PackageJsonDepValue;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use crate::util::path::is_banned_path_char;
#[derive(Debug)]
pub struct InstallNpmRemotePkg {
pub alias: String,
// todo(24419): use this when setting up the node_modules dir
#[allow(dead_code)]
pub base_dir: PathBuf,
pub req: PackageReq,
}
@ -19,74 +22,126 @@ pub struct InstallNpmRemotePkg {
#[derive(Debug)]
pub struct InstallNpmWorkspacePkg {
pub alias: String,
// todo(24419): use this when setting up the node_modules dir
#[allow(dead_code)]
pub base_dir: PathBuf,
pub target_dir: PathBuf,
}
#[derive(Debug, Default)]
pub struct PackageJsonInstallDepsProvider {
pub struct NpmInstallDepsProvider {
remote_pkgs: Vec<InstallNpmRemotePkg>,
workspace_pkgs: Vec<InstallNpmWorkspacePkg>,
}
impl PackageJsonInstallDepsProvider {
impl NpmInstallDepsProvider {
pub fn empty() -> Self {
Self::default()
}
pub fn from_workspace(workspace: &Arc<Workspace>) -> Self {
// todo(dsherret): estimate capacity?
let mut workspace_pkgs = Vec::new();
let mut remote_pkgs = Vec::new();
let workspace_npm_pkgs = workspace.npm_packages();
for pkg_json in workspace.package_jsons() {
let deps = pkg_json.resolve_local_package_json_deps();
let mut pkg_pkgs = Vec::with_capacity(deps.len());
for (alias, dep) in deps {
let Ok(dep) = dep else {
continue;
};
match dep {
PackageJsonDepValue::Req(pkg_req) => {
let workspace_pkg = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_req(&pkg_req)
// do not resolve to the current package
&& pkg.pkg_json.path != pkg_json.path
});
for (_, folder) in workspace.config_folders() {
let mut deno_json_aliases = HashSet::new();
// deal with the deno.json first because it takes precedence during resolution
if let Some(deno_json) = &folder.deno_json {
// don't bother with externally referenced import maps as users
// should inline their import map to get this behaviour
if let Some(serde_json::Value::Object(obj)) = &deno_json.json.imports {
deno_json_aliases.reserve(obj.len());
let mut pkg_pkgs = Vec::with_capacity(obj.len());
for (alias, value) in obj {
let serde_json::Value::String(specifier) = value else {
continue;
};
let Ok(npm_req_ref) = NpmPackageReqReference::from_str(specifier)
else {
continue;
};
// skip any aliases with banned characters
if alias.chars().any(|c| c == '\\' || is_banned_path_char(c)) {
continue;
}
deno_json_aliases.insert(alias.to_lowercase());
let pkg_req = npm_req_ref.into_inner().req;
let workspace_pkg = workspace_npm_pkgs
.iter()
.find(|pkg| pkg.matches_req(&pkg_req));
if let Some(pkg) = workspace_pkg {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias,
base_dir: pkg_json.dir_path().to_path_buf(),
alias: alias.to_string(),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
} else {
pkg_pkgs.push(InstallNpmRemotePkg {
alias,
base_dir: pkg_json.dir_path().to_path_buf(),
alias: alias.to_string(),
base_dir: deno_json.dir_path(),
req: pkg_req,
});
}
}
PackageJsonDepValue::Workspace(version_req) => {
if let Some(pkg) = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_name_and_version_req(&alias, &version_req)
}) {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias,
base_dir: pkg_json.dir_path().to_path_buf(),
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
// sort within each package (more like npm resolution)
pkg_pkgs.sort_by(|a, b| a.alias.cmp(&b.alias));
remote_pkgs.extend(pkg_pkgs);
}
}
if let Some(pkg_json) = &folder.pkg_json {
let deps = pkg_json.resolve_local_package_json_deps();
let mut pkg_pkgs = Vec::with_capacity(deps.len());
for (alias, dep) in deps {
let Ok(dep) = dep else {
continue;
};
if deno_json_aliases.contains(&alias.to_lowercase()) {
// aliases in deno.json take precedence over package.json, so
// since this can't be resolved don't bother installing it
continue;
}
match dep {
PackageJsonDepValue::Req(pkg_req) => {
let workspace_pkg = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_req(&pkg_req)
// do not resolve to the current package
&& pkg.pkg_json.path != pkg_json.path
});
if let Some(pkg) = workspace_pkg {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias,
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
} else {
pkg_pkgs.push(InstallNpmRemotePkg {
alias,
base_dir: pkg_json.dir_path().to_path_buf(),
req: pkg_req,
});
}
}
PackageJsonDepValue::Workspace(version_req) => {
if let Some(pkg) = workspace_npm_pkgs.iter().find(|pkg| {
pkg.matches_name_and_version_req(&alias, &version_req)
}) {
workspace_pkgs.push(InstallNpmWorkspacePkg {
alias,
target_dir: pkg.pkg_json.dir_path().to_path_buf(),
});
}
}
}
}
}
// sort within each package
pkg_pkgs.sort_by(|a, b| a.alias.cmp(&b.alias));
remote_pkgs.extend(pkg_pkgs);
// sort within each package as npm does
pkg_pkgs.sort_by(|a, b| a.alias.cmp(&b.alias));
remote_pkgs.extend(pkg_pkgs);
}
}
remote_pkgs.shrink_to_fit();
workspace_pkgs.shrink_to_fit();
Self {

6
cli/factory.rs
View file

@ -5,7 +5,7 @@ use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::Flags;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
@ -386,9 +386,7 @@ impl CliFactory {
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options.node_modules_dir_path().cloned(),
package_json_deps_provider: Arc::new(PackageJsonInstallDepsProvider::from_workspace(
cli_options.workspace(),
)),
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::from_workspace(cli_options.workspace())),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),

12
cli/lsp/documents.rs
View file

@ -1251,7 +1251,7 @@ impl Documents {
/// tsc when type checking.
pub fn resolve(
&self,
specifiers: &[String],
raw_specifiers: &[String],
referrer: &ModuleSpecifier,
file_referrer: Option<&ModuleSpecifier>,
) -> Vec<Option<(ModuleSpecifier, MediaType)>> {
@ -1262,16 +1262,16 @@ impl Documents {
.or(file_referrer);
let dependencies = document.as_ref().map(|d| d.dependencies());
let mut results = Vec::new();
for specifier in specifiers {
if specifier.starts_with("asset:") {
if let Ok(specifier) = ModuleSpecifier::parse(specifier) {
for raw_specifier in raw_specifiers {
if raw_specifier.starts_with("asset:") {
if let Ok(specifier) = ModuleSpecifier::parse(raw_specifier) {
let media_type = MediaType::from_specifier(&specifier);
results.push(Some((specifier, media_type)));
} else {
results.push(None);
}
} else if let Some(dep) =
dependencies.as_ref().and_then(|d| d.get(specifier))
dependencies.as_ref().and_then(|d| d.get(raw_specifier))
{
if let Some(specifier) = dep.maybe_type.maybe_specifier() {
results.push(self.resolve_dependency(
@ -1290,7 +1290,7 @@ impl Documents {
}
} else if let Ok(specifier) =
self.resolver.as_graph_resolver(file_referrer).resolve(
specifier,
raw_specifier,
&deno_graph::Range {
specifier: referrer.clone(),
start: deno_graph::Position::zeroed(),

6
cli/lsp/resolver.rs
View file

@ -3,7 +3,7 @@
use crate::args::create_default_npmrc;
use crate::args::CacheSetting;
use crate::args::CliLockfile;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::graph_util::CliJsrUrlProvider;
use crate::http_util::HttpClientProvider;
use crate::lsp::config::Config;
@ -474,9 +474,7 @@ async fn create_npm_resolver(
maybe_node_modules_path: config_data
.and_then(|d| d.node_modules_dir.clone()),
// only used for top level install, so we can ignore this
package_json_deps_provider: Arc::new(
PackageJsonInstallDepsProvider::empty(),
),
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
npmrc: config_data
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc),

8
cli/module_loader.rs
View file

@ -401,7 +401,7 @@ impl<TGraphContainer: ModuleGraphContainer>
fn inner_resolve(
&self,
specifier: &str,
raw_specifier: &str,
referrer: &ModuleSpecifier,
) -> Result<ModuleSpecifier, AnyError> {
if self.shared.node_resolver.in_npm_package(referrer) {
@ -409,7 +409,7 @@ impl<TGraphContainer: ModuleGraphContainer>
self
.shared
.node_resolver
.resolve(specifier, referrer, NodeResolutionMode::Execution)?
.resolve(raw_specifier, referrer, NodeResolutionMode::Execution)?
.into_url(),
);
}
@ -418,7 +418,7 @@ impl<TGraphContainer: ModuleGraphContainer>
let resolution = match graph.get(referrer) {
Some(Module::Js(module)) => module
.dependencies
.get(specifier)
.get(raw_specifier)
.map(|d| &d.maybe_code)
.unwrap_or(&Resolution::None),
_ => &Resolution::None,
@ -433,7 +433,7 @@ impl<TGraphContainer: ModuleGraphContainer>
));
}
Resolution::None => Cow::Owned(self.shared.resolver.resolve(
specifier,
raw_specifier,
&deno_graph::Range {
specifier: referrer.clone(),
start: deno_graph::Position::zeroed(),

143
cli/npm/byonm.rs
View file

@ -17,6 +17,7 @@ use deno_runtime::deno_node::NodeRequireResolver;
use deno_runtime::deno_node::NpmProcessStateProvider;
use deno_runtime::deno_node::PackageJson;
use deno_semver::package::PackageReq;
use deno_semver::Version;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageJsonLoadError;
@ -29,6 +30,7 @@ use crate::args::NpmProcessStateKind;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use deno_runtime::fs_util::specifier_to_file_path;
use super::managed::normalize_pkg_name_for_node_modules_deno_folder;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
@ -60,9 +62,7 @@ impl ByonmCliNpmResolver {
) -> Result<Option<Arc<PackageJson>>, PackageJsonLoadError> {
load_pkg_json(&DenoPkgJsonFsAdapter(self.fs.as_ref()), path)
}
}
impl ByonmCliNpmResolver {
/// Finds the ancestor package.json that contains the specified dependency.
pub fn find_ancestor_package_json_with_dep(
&self,
@ -98,7 +98,7 @@ impl ByonmCliNpmResolver {
&self,
req: &PackageReq,
referrer: &ModuleSpecifier,
) -> Result<(Arc<PackageJson>, String), AnyError> {
) -> Result<Option<(Arc<PackageJson>, String)>, AnyError> {
fn resolve_alias_from_pkg_json(
req: &PackageReq,
pkg_json: &PackageJson,
@ -134,7 +134,7 @@ impl ByonmCliNpmResolver {
if let Some(alias) =
resolve_alias_from_pkg_json(req, pkg_json.as_ref())
{
return Ok((pkg_json, alias));
return Ok(Some((pkg_json, alias)));
}
}
current_path = dir_path;
@ -148,19 +148,65 @@ impl ByonmCliNpmResolver {
if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? {
if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref())
{
return Ok((pkg_json, alias));
return Ok(Some((pkg_json, alias)));
}
}
}
bail!(
concat!(
"Could not find a matching package for 'npm:{}' in a package.json file. ",
"You must specify this as a package.json dependency when the ",
"node_modules folder is not managed by Deno.",
),
req,
Ok(None)
}
fn resolve_folder_in_root_node_modules(
&self,
req: &PackageReq,
) -> Option<PathBuf> {
// now check if node_modules/.deno/ matches this constraint
let root_node_modules_dir = self.root_node_modules_dir.as_ref()?;
let node_modules_deno_dir = root_node_modules_dir.join(".deno");
let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else {
return None;
};
let search_prefix = format!(
"{}@",
normalize_pkg_name_for_node_modules_deno_folder(&req.name)
);
let mut best_version = None;
// example entries:
// - @denotest+add@1.0.0
// - @denotest+add@1.0.0_1
for entry in entries {
if !entry.is_directory {
continue;
}
let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix)
else {
continue;
};
let version = version_and_copy_idx
.rsplit_once('_')
.map(|(v, _)| v)
.unwrap_or(version_and_copy_idx);
let Ok(version) = Version::parse_from_npm(version) else {
continue;
};
if req.version_req.matches(&version) {
if let Some((best_version_version, _)) = &best_version {
if version > *best_version_version {
best_version = Some((version, entry.name));
}
} else {
best_version = Some((version, entry.name));
}
}
}
best_version.map(|(_version, entry_name)| {
join_package_name(
&node_modules_deno_dir.join(entry_name).join("node_modules"),
&req.name,
)
})
}
}
@ -288,29 +334,62 @@ impl CliNpmResolver for ByonmCliNpmResolver {
req: &PackageReq,
referrer: &ModuleSpecifier,
) -> Result<PathBuf, AnyError> {
// resolve the pkg json and alias
let (pkg_json, alias) =
self.resolve_pkg_json_and_alias_for_req(req, referrer)?;
// now try node resolution
for ancestor in pkg_json.path.parent().unwrap().ancestors() {
let node_modules_folder = ancestor.join("node_modules");
let sub_dir = join_package_name(&node_modules_folder, &alias);
if self.fs.is_dir_sync(&sub_dir) {
return Ok(canonicalize_path_maybe_not_exists_with_fs(
&sub_dir,
self.fs.as_ref(),
)?);
fn node_resolve_dir(
fs: &dyn FileSystem,
alias: &str,
start_dir: &Path,
) -> Result<Option<PathBuf>, AnyError> {
for ancestor in start_dir.ancestors() {
let node_modules_folder = ancestor.join("node_modules");
let sub_dir = join_package_name(&node_modules_folder, alias);
if fs.is_dir_sync(&sub_dir) {
return Ok(Some(canonicalize_path_maybe_not_exists_with_fs(
&sub_dir, fs,
)?));
}
}
Ok(None)
}
bail!(
concat!(
"Could not find \"{}\" in a node_modules folder. ",
"Deno expects the node_modules/ directory to be up to date. ",
"Did you forget to run `deno install`?"
),
alias,
);
// now attempt to resolve if it's found in any package.json
let maybe_pkg_json_and_alias =
self.resolve_pkg_json_and_alias_for_req(req, referrer)?;
match maybe_pkg_json_and_alias {
Some((pkg_json, alias)) => {
// now try node resolution
if let Some(resolved) =
node_resolve_dir(self.fs.as_ref(), &alias, pkg_json.dir_path())?
{
return Ok(resolved);
}
bail!(
concat!(
"Could not find \"{}\" in a node_modules folder. ",
"Deno expects the node_modules/ directory to be up to date. ",
"Did you forget to run `deno install`?"
),
alias,
);
}
None => {
// now check if node_modules/.deno/ matches this constraint
if let Some(folder) = self.resolve_folder_in_root_node_modules(req) {
return Ok(folder);
}
bail!(
concat!(
"Could not find a matching package for 'npm:{}' in the node_modules ",
"directory. Ensure you have all your JSR and npm dependencies listed ",
"in your deno.json or package.json, then run `deno install`. Alternatively, ",
r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#,
"deno.json file."
),
req,
);
}
}
}
fn check_state_hash(&self) -> Option<u64> {

27
cli/npm/managed/mod.rs
View file

@ -32,9 +32,9 @@ use resolution::AddPkgReqsResult;
use crate::args::CliLockfile;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonInstallDepsProvider;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
@ -45,6 +45,7 @@ use self::cache::NpmCache;
use self::registry::CliNpmRegistryApi;
use self::resolution::NpmResolution;
use self::resolvers::create_npm_fs_resolver;
pub use self::resolvers::normalize_pkg_name_for_node_modules_deno_folder;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmResolver;
@ -71,7 +72,7 @@ pub struct CliNpmResolverManagedCreateOptions {
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
pub npm_system_info: NpmSystemInfo,
pub package_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
pub npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
pub npmrc: Arc<ResolvedNpmRc>,
pub lifecycle_scripts: LifecycleScriptsConfig,
}
@ -97,7 +98,7 @@ pub async fn create_managed_npm_resolver_for_lsp(
npm_api,
npm_cache,
options.npmrc,
options.package_json_deps_provider,
options.npm_install_deps_provider,
options.text_only_progress_bar,
options.maybe_node_modules_path,
options.npm_system_info,
@ -122,7 +123,7 @@ pub async fn create_managed_npm_resolver(
npm_api,
npm_cache,
options.npmrc,
options.package_json_deps_provider,
options.npm_install_deps_provider,
options.text_only_progress_bar,
options.maybe_node_modules_path,
options.npm_system_info,
@ -139,7 +140,7 @@ fn create_inner(
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
package_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
node_modules_dir_path: Option<PathBuf>,
npm_system_info: NpmSystemInfo,
@ -161,7 +162,7 @@ fn create_inner(
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache.clone(),
&package_json_deps_provider,
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
tarball_cache.clone(),
@ -175,7 +176,7 @@ fn create_inner(
maybe_lockfile,
npm_api,
npm_cache,
package_json_deps_provider,
npm_install_deps_provider,
resolution,
tarball_cache,
text_only_progress_bar,
@ -261,7 +262,7 @@ pub struct ManagedCliNpmResolver {
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
package_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
text_only_progress_bar: ProgressBar,
@ -286,7 +287,7 @@ impl ManagedCliNpmResolver {
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_api: Arc<CliNpmRegistryApi>,
npm_cache: Arc<NpmCache>,
package_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
text_only_progress_bar: ProgressBar,
@ -299,7 +300,7 @@ impl ManagedCliNpmResolver {
maybe_lockfile,
npm_api,
npm_cache,
package_json_deps_provider,
npm_install_deps_provider,
text_only_progress_bar,
resolution,
tarball_cache,
@ -476,7 +477,7 @@ impl ManagedCliNpmResolver {
if !self.top_level_install_flag.raise() {
return Ok(false); // already did this
}
let pkg_json_remote_pkgs = self.package_json_deps_provider.remote_pkgs();
let pkg_json_remote_pkgs = self.npm_install_deps_provider.remote_pkgs();
if pkg_json_remote_pkgs.is_empty() {
return Ok(false);
}
@ -605,7 +606,7 @@ impl CliNpmResolver for ManagedCliNpmResolver {
create_npm_fs_resolver(
self.fs.clone(),
self.npm_cache.clone(),
&self.package_json_deps_provider,
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
npm_resolution.clone(),
self.tarball_cache.clone(),
@ -616,7 +617,7 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self.maybe_lockfile.clone(),
self.npm_api.clone(),
self.npm_cache.clone(),
self.package_json_deps_provider.clone(),
self.npm_install_deps_provider.clone(),
npm_resolution,
self.tarball_cache.clone(),
self.text_only_progress_bar.clone(),

50
cli/npm/managed/resolvers/local.rs
View file

@ -41,7 +41,7 @@ use node_resolver::errors::ReferrerNotFoundError;
use serde::Deserialize;
use serde::Serialize;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::npm::cache_dir::mixed_case_package_name_decode;
use crate::npm::cache_dir::mixed_case_package_name_encode;
@ -65,7 +65,7 @@ use super::common::RegistryReadPermissionChecker;
pub struct LocalNpmPackageResolver {
cache: Arc<NpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
pkg_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
@ -81,7 +81,7 @@ impl LocalNpmPackageResolver {
pub fn new(
cache: Arc<NpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
pkg_json_deps_provider: Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
@ -92,7 +92,7 @@ impl LocalNpmPackageResolver {
Self {
cache,
fs: fs.clone(),
pkg_json_deps_provider,
npm_install_deps_provider,
progress_bar,
resolution,
tarball_cache,
@ -248,7 +248,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
sync_resolution_with_fs(
&self.resolution.snapshot(),
&self.cache,
&self.pkg_json_deps_provider,
&self.npm_install_deps_provider,
&self.progress_bar,
&self.tarball_cache,
&self.root_node_modules_path,
@ -412,14 +412,16 @@ fn has_lifecycle_scripts(
async fn sync_resolution_with_fs(
snapshot: &NpmResolutionSnapshot,
cache: &Arc<NpmCache>,
pkg_json_deps_provider: &PackageJsonInstallDepsProvider,
npm_install_deps_provider: &NpmInstallDepsProvider,
progress_bar: &ProgressBar,
tarball_cache: &Arc<TarballCache>,
root_node_modules_dir_path: &Path,
system_info: &NpmSystemInfo,
lifecycle_scripts: &LifecycleScriptsConfig,
) -> Result<(), AnyError> {
if snapshot.is_empty() && pkg_json_deps_provider.workspace_pkgs().is_empty() {
if snapshot.is_empty()
&& npm_install_deps_provider.workspace_pkgs().is_empty()
{
return Ok(()); // don't create the directory
}
@ -620,7 +622,7 @@ async fn sync_resolution_with_fs(
// 4. Create symlinks for package json dependencies
{
for remote in pkg_json_deps_provider.remote_pkgs() {
for remote in npm_install_deps_provider.remote_pkgs() {
let remote_pkg = if let Ok(remote_pkg) =
snapshot.resolve_pkg_from_pkg_req(&remote.req)
{
@ -684,7 +686,7 @@ async fn sync_resolution_with_fs(
}
// 5. Create symlinks for the remaining top level packages in the node_modules folder.
// (These may be present if they are not in the package.json dependencies, such as )
// (These may be present if they are not in the package.json dependencies)
// Symlink node_modules/.deno/<package_id>/node_modules/<package_name> to
// node_modules/<package_name>
let mut ids = snapshot
@ -757,10 +759,10 @@ async fn sync_resolution_with_fs(
// 8. Create symlinks for the workspace packages
{
// todo(#24419): this is not exactly correct because it should
// todo(dsherret): this is not exactly correct because it should
// install correctly for a workspace (potentially in sub directories),
// but this is good enough for a first pass
for workspace in pkg_json_deps_provider.workspace_pkgs() {
for workspace in npm_install_deps_provider.workspace_pkgs() {
symlink_package_dir(
&workspace.target_dir,
&root_node_modules_dir_path.join(&workspace.alias),
@ -985,21 +987,31 @@ impl SetupCache {
}
}
/// Normalizes a package name for use at `node_modules/.deno/<pkg-name>@<version>[_<copy_index>]`
pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow<str> {
let name = if name.to_lowercase() == name {
Cow::Borrowed(name)
} else {
Cow::Owned(format!("_{}", mixed_case_package_name_encode(name)))
};
if name.starts_with('@') {
name.replace('/', "+").into()
} else {
name
}
}
fn get_package_folder_id_folder_name(
folder_id: &NpmPackageCacheFolderId,
) -> String {
let copy_str = if folder_id.copy_index == 0 {
"".to_string()
Cow::Borrowed("")
} else {
format!("_{}", folder_id.copy_index)
Cow::Owned(format!("_{}", folder_id.copy_index))
};
let nv = &folder_id.nv;
let name = if nv.name.to_lowercase() == nv.name {
Cow::Borrowed(&nv.name)
} else {
Cow::Owned(format!("_{}", mixed_case_package_name_encode(&nv.name)))
};
format!("{}@{}{}", name, nv.version, copy_str).replace('/', "+")
let name = normalize_pkg_name_for_node_modules_deno_folder(&nv.name);
format!("{}@{}{}", name, nv.version, copy_str)
}
fn get_package_folder_id_from_folder_name(

7
cli/npm/managed/resolvers/mod.rs
View file

@ -11,10 +11,11 @@ use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use crate::args::LifecycleScriptsConfig;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::util::progress_bar::ProgressBar;
pub use self::common::NpmPackageFsResolver;
pub use self::local::normalize_pkg_name_for_node_modules_deno_folder;
use self::global::GlobalNpmPackageResolver;
use self::local::LocalNpmPackageResolver;
@ -27,7 +28,7 @@ use super::resolution::NpmResolution;
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<NpmCache>,
pkg_json_deps_provider: &Arc<PackageJsonInstallDepsProvider>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<TarballCache>,
@ -39,7 +40,7 @@ pub fn create_npm_fs_resolver(
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
npm_cache,
fs,
pkg_json_deps_provider.clone(),
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
tarball_cache,

8
cli/resolver.rs
View file

@ -502,7 +502,7 @@ impl Resolver for CliGraphResolver {
fn resolve(
&self,
specifier: &str,
raw_specifier: &str,
referrer_range: &deno_graph::Range,
mode: ResolutionMode,
) -> Result<ModuleSpecifier, ResolveError> {
@ -519,7 +519,7 @@ impl Resolver for CliGraphResolver {
if let Some(node_resolver) = self.node_resolver.as_ref() {
if referrer.scheme() == "file" && node_resolver.in_npm_package(referrer) {
return node_resolver
.resolve(specifier, referrer, to_node_mode(mode))
.resolve(raw_specifier, referrer, to_node_mode(mode))
.map(|res| res.into_url())
.map_err(|e| ResolveError::Other(e.into()));
}
@ -528,7 +528,7 @@ impl Resolver for CliGraphResolver {
// Attempt to resolve with the workspace resolver
let result: Result<_, ResolveError> = self
.workspace_resolver
.resolve(specifier, referrer)
.resolve(raw_specifier, referrer)
.map_err(|err| match err {
MappedResolutionError::Specifier(err) => ResolveError::Specifier(err),
MappedResolutionError::ImportMap(err) => {
@ -700,7 +700,7 @@ impl Resolver for CliGraphResolver {
// If byonm, check if the bare specifier resolves to an npm package
if is_byonm && referrer.scheme() == "file" {
let maybe_resolution = node_resolver
.resolve_if_for_npm_pkg(specifier, referrer, to_node_mode(mode))
.resolve_if_for_npm_pkg(raw_specifier, referrer, to_node_mode(mode))
.map_err(ResolveError::Other)?;
if let Some(res) = maybe_resolution {
return Ok(res.into_url());

2
cli/standalone/binary.rs
View file

@ -45,7 +45,7 @@ use serde::Serialize;
use crate::args::CaData;
use crate::args::CliOptions;
use crate::args::CompileFlags;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::args::PermissionFlags;
use crate::args::UnstableConfig;
use crate::cache::DenoDir;

22
cli/standalone/mod.rs
View file

@ -48,7 +48,7 @@ use crate::args::get_root_cert_store;
use crate::args::npm_pkg_req_ref_to_binary_command;
use crate::args::CaData;
use crate::args::CacheSetting;
use crate::args::PackageJsonInstallDepsProvider;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoDirProvider;
@ -138,7 +138,7 @@ pub const UNSUPPORTED_SCHEME: &str = "Unsupported scheme";
impl ModuleLoader for EmbeddedModuleLoader {
fn resolve(
&self,
specifier: &str,
raw_specifier: &str,
referrer: &str,
kind: ResolutionKind,
) -> Result<ModuleSpecifier, AnyError> {
@ -162,13 +162,15 @@ impl ModuleLoader for EmbeddedModuleLoader {
self
.shared
.node_resolver
.resolve(specifier, &referrer, NodeResolutionMode::Execution)?
.resolve(raw_specifier, &referrer, NodeResolutionMode::Execution)?
.into_url(),
);
}
let mapped_resolution =
self.shared.workspace_resolver.resolve(specifier, &referrer);
let mapped_resolution = self
.shared
.workspace_resolver
.resolve(raw_specifier, &referrer);
match mapped_resolution {
Ok(MappedResolution::WorkspaceJsrPackage { specifier, .. }) => {
@ -262,7 +264,7 @@ impl ModuleLoader for EmbeddedModuleLoader {
if err.is_unmapped_bare_specifier() && referrer.scheme() == "file" =>
{
let maybe_res = self.shared.node_resolver.resolve_if_for_npm_pkg(
specifier,
raw_specifier,
&referrer,
NodeResolutionMode::Execution,
)?;
@ -502,9 +504,9 @@ pub async fn run(
text_only_progress_bar: progress_bar,
maybe_node_modules_path,
npm_system_info: Default::default(),
package_json_deps_provider: Arc::new(
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
PackageJsonInstallDepsProvider::empty(),
NpmInstallDepsProvider::empty(),
),
// create an npmrc that uses the fake npm_registry_url to resolve packages
npmrc: Arc::new(ResolvedNpmRc {
@ -554,9 +556,9 @@ pub async fn run(
text_only_progress_bar: progress_bar,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
package_json_deps_provider: Arc::new(
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
PackageJsonInstallDepsProvider::empty(),
NpmInstallDepsProvider::empty(),
),
// Packages from different registries are already inlined in the ESZip,
// so no need to create actual `.npmrc` configuration.

8
cli/tsc/mod.rs
View file

@ -795,7 +795,7 @@ fn resolve_graph_specifier_types(
}
fn resolve_non_graph_specifier_types(
specifier: &str,
raw_specifier: &str,
referrer: &ModuleSpecifier,
referrer_kind: NodeModuleKind,
state: &State,
@ -810,14 +810,16 @@ fn resolve_non_graph_specifier_types(
Ok(Some(NodeResolution::into_specifier_and_media_type(
node_resolver
.resolve(
specifier,
raw_specifier,
referrer,
referrer_kind,
NodeResolutionMode::Types,
)
.ok(),
)))
} else if let Ok(npm_req_ref) = NpmPackageReqReference::from_str(specifier) {
} else if let Ok(npm_req_ref) =
NpmPackageReqReference::from_str(raw_specifier)
{
debug_assert_eq!(referrer_kind, NodeModuleKind::Esm);
// todo(dsherret): add support for injecting this in the graph so
// we don't need this special code here.

1
ext/node_resolver/lib.rs
View file

@ -18,6 +18,7 @@ pub use npm::NpmResolverRc;
pub use package_json::load_pkg_json;
pub use package_json::PackageJsonThreadLocalCache;
pub use path::PathClean;
pub use resolution::parse_npm_pkg_name;
pub use resolution::NodeModuleKind;
pub use resolution::NodeResolution;
pub use resolution::NodeResolutionMode;

4
tests/integration/lsp_tests.rs
View file

@ -14797,7 +14797,7 @@ fn lsp_byonm() {
"severity": 1,
"code": "resolver-error",
"source": "deno",
"message": "Could not find a matching package for 'npm:chalk' in a package.json file. You must specify this as a package.json dependency when the node_modules folder is not managed by Deno.",
"message": "Could not find a matching package for 'npm:chalk' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `\"nodeModulesDir\": \"auto\"` in your deno.json file.",
},
{
"range": {
@ -14842,7 +14842,7 @@ fn lsp_byonm() {
"severity": 1,
"code": "resolver-error",
"source": "deno",
"message": "Could not find a matching package for 'npm:chalk' in a package.json file. You must specify this as a package.json dependency when the node_modules folder is not managed by Deno.",
"message": "Could not find a matching package for 'npm:chalk' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `\"nodeModulesDir\": \"auto\"` in your deno.json file.",
},
])
);

4
tests/integration/npm_tests.rs
View file

@ -2195,7 +2195,7 @@ console.log(getKind());
.args("run --allow-read chalk.ts")
.run();
output.assert_matches_text(
r#"error: Could not find a matching package for 'npm:chalk@5' in a package.json file. You must specify this as a package.json dependency when the node_modules folder is not managed by Deno.
r#"error: Could not find a matching package for 'npm:chalk@5' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file.
at file:///[WILDCARD]chalk.ts:1:19
"#);
output.assert_exit_code(1);
@ -2277,7 +2277,7 @@ console.log(getKind());
.args("run --allow-read chalk.ts")
.run();
output.assert_matches_text(
r#"error: Could not find a matching package for 'npm:chalk@5' in a package.json file. You must specify this as a package.json dependency when the node_modules folder is not managed by Deno.
r#"error: Could not find a matching package for 'npm:chalk@5' in the node_modules directory. Ensure you have all your JSR and npm dependencies listed in your deno.json or package.json, then run `deno install`. Alternatively, turn on auto-install by specifying `"nodeModulesDir": "auto"` in your deno.json file.
at file:///[WILDCARD]chalk.ts:1:19
"#);
output.assert_exit_code(1);

5
tests/registry/jsr/@denotest/npm-add/0.5.0/mod.ts Normal file
View file

@ -0,0 +1,5 @@
import * as npmAdd from "npm:@denotest/add@0.5";
export function sum(a: number, b: number): number {
return npmAdd.sum(a, b);
}

5
tests/registry/jsr/@denotest/npm-add/0.5.0_meta.json Normal file
View file

@ -0,0 +1,5 @@
{
"exports": {
".": "./mod.ts"
}
}

5
tests/registry/jsr/@denotest/npm-add/1.0.0/mod.ts Normal file
View file

@ -0,0 +1,5 @@
import * as npmAdd from "npm:@denotest/add@1";
export function add(a: number, b: number): number {
return npmAdd.add(a, b);
}

5
tests/registry/jsr/@denotest/npm-add/1.0.0_meta.json Normal file
View file

@ -0,0 +1,5 @@
{
"exports": {
".": "./mod.ts"
}
}

6
tests/registry/jsr/@denotest/npm-add/meta.json Normal file
View file

@ -0,0 +1,6 @@
{
"versions": {
"0.5.0": {},
"1.0.0": {}
}
}

10
tests/specs/install/alias_deno_json/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run --allow-read=. verify.ts",
"output": "true\n"
}]
}

5
tests/specs/install/alias_deno_json/deno.json Normal file
View file

@ -0,0 +1,5 @@
{
"imports": {
"alias": "npm:@denotest/add"
}
}

2
tests/specs/install/alias_deno_json/package.json Normal file
View file

@ -0,0 +1,2 @@
{
}

2
tests/specs/install/alias_deno_json/verify.ts Normal file
View file

@ -0,0 +1,2 @@
const stat = Deno.statSync(new URL("./node_modules/alias", import.meta.url));
console.log(stat.isDirectory);

10
tests/specs/install/alias_invalid_path_char/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run --allow-read=. verify.ts",
"output": ".bin\n.deno\n@denotest\n"
}]
}

7
tests/specs/install/alias_invalid_path_char/deno.jsonc Normal file
View file

@ -0,0 +1,7 @@
{
"imports": {
// alias*test is an invalid path char on windows, so
// don't create an alias for this
"alias*test": "npm:@denotest/add"
}
}

2
tests/specs/install/alias_invalid_path_char/package.json Normal file
View file

@ -0,0 +1,2 @@
{
}

10
tests/specs/install/alias_invalid_path_char/verify.ts Normal file
View file

@ -0,0 +1,10 @@
const entries = Array.from(
Deno.readDirSync(new URL("./node_modules", import.meta.url)),
);
const names = entries.map((entry) => entry.name);
names.sort();
// won't have the invalid path alias
for (const name of names) {
console.log(name);
}

10
tests/specs/install/alias_pkg_json_and_deno_json_jsr_pkg/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run --allow-read=. verify.ts",
"output": "verify.out"
}]
}

5
tests/specs/install/alias_pkg_json_and_deno_json_jsr_pkg/deno.json Normal file
View file

@ -0,0 +1,5 @@
{
"imports": {
"alias": "jsr:@denotest/add"
}
}

5
tests/specs/install/alias_pkg_json_and_deno_json_jsr_pkg/package.json Normal file
View file

@ -0,0 +1,5 @@
{
"dependencies": {
"alias": "npm:@denotest/esm-basic"
}
}

2
tests/specs/install/alias_pkg_json_and_deno_json_jsr_pkg/verify.out Normal file
View file

@ -0,0 +1,2 @@
@denotest/esm-basic
[Module: null prototype] { add: [Function: add] }

13
tests/specs/install/alias_pkg_json_and_deno_json_jsr_pkg/verify.ts Normal file
View file

@ -0,0 +1,13 @@
import * as mod from "alias";
const data = JSON.parse(
Deno.readTextFileSync(
new URL("./node_modules/alias/package.json", import.meta.url),
),
);
// this should just setup the npm package anyway, even though the alias
// will resolve to the jsr package
console.log(data.name);
console.log(mod);

10
tests/specs/install/alias_pkg_json_and_deno_json_npm_pkg/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run --allow-read=. verify.ts",
"output": "verify.out"
}]
}

5
tests/specs/install/alias_pkg_json_and_deno_json_npm_pkg/deno.json Normal file
View file

@ -0,0 +1,5 @@
{
"imports": {
"alias": "npm:@denotest/add"
}
}

5
tests/specs/install/alias_pkg_json_and_deno_json_npm_pkg/package.json Normal file
View file

@ -0,0 +1,5 @@
{
"dependencies": {
"alias": "npm:@denotest/esm-basic"
}
}

5
tests/specs/install/alias_pkg_json_and_deno_json_npm_pkg/verify.out Normal file
View file

@ -0,0 +1,5 @@
@denotest/add
[Module: null prototype] {
add: [Function (anonymous)],
default: { add: [Function (anonymous)] }
}

10
tests/specs/install/alias_pkg_json_and_deno_json_npm_pkg/verify.ts Normal file
View file

@ -0,0 +1,10 @@
import * as mod from "alias";
const data = JSON.parse(
Deno.readTextFileSync(
new URL("./node_modules/alias/package.json", import.meta.url),
),
);
console.log(data.name);
console.log(mod);

10
tests/specs/install/byonm_jsr_npm_dep/__test__.jsonc Normal file
View file

@ -0,0 +1,10 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run main.ts",
"output": "3\n4\n"
}]
}

6
tests/specs/install/byonm_jsr_npm_dep/deno.json Normal file
View file

@ -0,0 +1,6 @@
{
"imports": {
"@denotest/npm-add": "jsr:@denotest/npm-add@1",
"@denotest/npm-add-0-5": "jsr:@denotest/npm-add@0.5"
}
}

5
tests/specs/install/byonm_jsr_npm_dep/main.ts Normal file
View file

@ -0,0 +1,5 @@
import { add } from "@denotest/npm-add";
import { sum } from "@denotest/npm-add-0-5";
console.log(add(1, 2));
console.log(sum(2, 2));

2
tests/specs/install/byonm_jsr_npm_dep/package.json Normal file
View file

@ -0,0 +1,2 @@
{
}

13
tests/specs/workspaces/nested_deno_pkg_npm_conflict/__test__.jsonc Normal file
View file

@ -0,0 +1,13 @@
{
"tempDir": true,
"steps": [{
"args": "install",
"output": "[WILDCARD]"
}, {
"args": "run --allow-read main.js",
"output": "4\n0.5.0\n"
}, {
"args": "run --allow-read member/main.js",
"output": "3\n1.0.0\n"
}]
}

7
tests/specs/workspaces/nested_deno_pkg_npm_conflict/deno.json Normal file
View file

@ -0,0 +1,7 @@
{
"nodeModulesDir": "manual",
"workspace": ["./member"],
"imports": {
"@denotest/add": "npm:@denotest/add@0.5"
}
}

9
tests/specs/workspaces/nested_deno_pkg_npm_conflict/main.js Normal file
View file

@ -0,0 +1,9 @@
import { sum } from "@denotest/add";
console.log(sum(2, 2));
console.log(
JSON.parse(Deno.readTextFileSync(
new URL("node_modules/@denotest/add/package.json", import.meta.url),
)).version,
);

5
tests/specs/workspaces/nested_deno_pkg_npm_conflict/member/deno.json Normal file
View file

@ -0,0 +1,5 @@
{
"imports": {
"@denotest/add": "npm:@denotest/add@1"
}
}

9
tests/specs/workspaces/nested_deno_pkg_npm_conflict/member/main.js Normal file
View file

@ -0,0 +1,9 @@
import { add } from "@denotest/add";
console.log(add(1, 2));
console.log(
JSON.parse(Deno.readTextFileSync(
new URL("node_modules/@denotest/add/package.json", import.meta.url),
)).version,
);