diff --git a/Cargo.lock b/Cargo.lock index da8df04a2b..a638e2723f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -830,7 +830,6 @@ dependencies = [ "notify", "once_cell", "os_pipe", - "path-clean", "percent-encoding 2.2.0", "pin-project", "pretty_assertions", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index fa7284d54c..7fa0afa76e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -85,7 +85,6 @@ monch = "=0.2.0" notify = "=5.0.0" once_cell = "=1.14.0" os_pipe = "=1.0.1" -path-clean = "=0.1.0" percent-encoding = "=2.2.0" pin-project = "1.0.11" # don't pin because they yank crates from cargo rand = { version = "=0.8.5", features = ["small_rng"] } diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 7922a3c19d..513307e929 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -301,6 +301,7 @@ pub struct Flags { pub cached_only: bool, pub type_check_mode: TypeCheckMode, pub config_flag: ConfigFlag, + pub node_modules_dir: bool, pub coverage_dir: Option, pub enable_testing_features: bool, pub ignore: Vec, @@ -1734,6 +1735,7 @@ fn compile_args(app: Command) -> Command { .arg(import_map_arg()) .arg(no_remote_arg()) .arg(no_npm_arg()) + .arg(local_npm_arg()) .arg(no_config_arg()) .arg(config_arg()) .arg(no_check_arg()) @@ -1749,6 +1751,7 @@ fn compile_args_without_check_args(app: Command) -> Command { .arg(import_map_arg()) .arg(no_remote_arg()) .arg(no_npm_arg()) + .arg(local_npm_arg()) .arg(config_arg()) .arg(no_config_arg()) .arg(reload_arg()) @@ -2153,6 +2156,12 @@ fn no_npm_arg<'a>() -> Arg<'a> { .help("Do not resolve npm modules") } +fn local_npm_arg<'a>() -> Arg<'a> { + Arg::new("node-modules-dir") + .long("node-modules-dir") + .help("Creates a local node_modules folder") +} + fn unsafely_ignore_certificate_errors_arg<'a>() -> Arg<'a> { Arg::new("unsafely-ignore-certificate-errors") .long("unsafely-ignore-certificate-errors") @@ -2799,6 +2808,7 @@ fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) { import_map_arg_parse(flags, matches); no_remote_arg_parse(flags, matches); no_npm_arg_parse(flags, matches); + local_npm_args_parse(flags, matches); config_args_parse(flags, matches); no_check_arg_parse(flags, matches); check_arg_parse(flags, matches); @@ -2814,6 +2824,7 @@ fn compile_args_without_no_check_parse( import_map_arg_parse(flags, matches); no_remote_arg_parse(flags, matches); no_npm_arg_parse(flags, matches); + local_npm_args_parse(flags, matches); config_args_parse(flags, matches); reload_arg_parse(flags, matches); lock_args_parse(flags, matches); @@ -3061,6 +3072,12 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } } +fn local_npm_args_parse(flags: &mut Flags, matches: &ArgMatches) { + if matches.is_present("node-modules-dir") { + flags.node_modules_dir = true; + } +} + fn inspect_arg_validate(val: &str) -> Result<(), String> { match val.parse::() { Ok(_) => Ok(()), @@ -5034,6 +5051,22 @@ mod tests { ); } + #[test] + fn local_npm() { + let r = + flags_from_vec(svec!["deno", "run", "--node-modules-dir", "script.ts"]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Run(RunFlags { + script: "script.ts".to_string(), + }), + node_modules_dir: true, + ..Flags::default() + } + ); + } + #[test] fn cached_only() { let r = flags_from_vec(svec!["deno", "run", "--cached-only", "script.ts"]); diff --git a/cli/args/mod.rs b/cli/args/mod.rs index e4c0d85560..1e0c72b1c8 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -42,6 +42,7 @@ use crate::emit::TsConfigWithIgnoredOptions; use crate::emit::TsTypeLib; use crate::file_fetcher::get_root_cert_store; use crate::file_fetcher::CacheSetting; +use crate::fs_util; use crate::lockfile::Lockfile; use crate::version; @@ -146,6 +147,24 @@ impl CliOptions { self.overrides.import_map_specifier = Some(path); } + /// Resolves the path to use for a local node_modules folder. + pub fn resolve_local_node_modules_folder( + &self, + ) -> Result, AnyError> { + let path = if !self.flags.node_modules_dir { + return Ok(None); + } else if let Some(config_path) = self + .maybe_config_file + .as_ref() + .and_then(|c| c.specifier.to_file_path().ok()) + { + config_path.parent().unwrap().join("node_modules") + } else { + std::env::current_dir()?.join("node_modules") + }; + Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?)) + } + pub fn resolve_root_cert_store(&self) -> Result { get_root_cert_store( None, diff --git a/cli/fs_util.rs b/cli/fs_util.rs index f3a4addc03..365c9b4301 100644 --- a/cli/fs_util.rs +++ b/cli/fs_util.rs @@ -5,10 +5,11 @@ use deno_core::error::{uri_error, AnyError}; pub use deno_core::normalize_path; use deno_core::ModuleSpecifier; use deno_runtime::deno_crypto::rand; +use deno_runtime::deno_node::PathClean; use std::borrow::Cow; use std::env::current_dir; use std::fs::OpenOptions; -use std::io::{Error, Write}; +use std::io::{Error, ErrorKind, Write}; use std::path::{Path, PathBuf}; use walkdir::WalkDir; @@ -75,6 +76,35 @@ pub fn canonicalize_path(path: &Path) -> Result { return Ok(path); } +/// Canonicalizes a path which might be non-existent by going up the +/// ancestors until it finds a directory that exists, canonicalizes +/// that path, then adds back the remaining path components. +/// +/// Note: When using this, you should be aware that a symlink may +/// subsequently be created along this path by some other code. +pub fn canonicalize_path_maybe_not_exists( + path: &Path, +) -> Result { + let path = path.to_path_buf().clean(); + let mut path = path.as_path(); + let mut names_stack = Vec::new(); + loop { + match canonicalize_path(path) { + Ok(mut canonicalized_path) => { + for name in names_stack.into_iter().rev() { + canonicalized_path = canonicalized_path.join(name); + } + return Ok(canonicalized_path); + } + Err(err) if err.kind() == ErrorKind::NotFound => { + names_stack.push(path.file_name().unwrap()); + path = path.parent().unwrap(); + } + Err(err) => return Err(err), + } + } +} + #[cfg(windows)] fn strip_unc_prefix(path: PathBuf) -> PathBuf { use std::path::Component; @@ -294,6 +324,60 @@ pub async fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> { } } +/// Copies a directory to another directory. +/// +/// Note: Does not handle symlinks. +pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { + std::fs::create_dir_all(&to) + .with_context(|| format!("Creating {}", to.display()))?; + let read_dir = std::fs::read_dir(&from) + .with_context(|| format!("Reading {}", from.display()))?; + + for entry in read_dir { + let entry = entry?; + let file_type = entry.file_type()?; + let new_from = from.join(entry.file_name()); + let new_to = to.join(entry.file_name()); + + if file_type.is_dir() { + copy_dir_recursive(&new_from, &new_to).with_context(|| { + format!("Dir {} to {}", new_from.display(), new_to.display()) + })?; + } else if file_type.is_file() { + std::fs::copy(&new_from, &new_to).with_context(|| { + format!("Copying {} to {}", new_from.display(), new_to.display()) + })?; + } + } + + Ok(()) +} + +pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> { + let err_mapper = |err: Error| { + Error::new( + err.kind(), + format!( + "{}, symlink '{}' -> '{}'", + err, + oldpath.display(), + newpath.display() + ), + ) + }; + #[cfg(unix)] + { + use std::os::unix::fs::symlink; + symlink(&oldpath, &newpath).map_err(err_mapper)?; + } + #[cfg(not(unix))] + { + use std::os::windows::fs::symlink_dir; + symlink_dir(&oldpath, &newpath).map_err(err_mapper)?; + } + Ok(()) +} + /// Attempts to convert a specifier to a file path. By default, uses the Url /// crate's `to_file_path()` method, but falls back to try and resolve unix-style /// paths on Windows. diff --git a/cli/node/mod.rs b/cli/node/mod.rs index 2ec9295dd8..c265afdf6a 100644 --- a/cli/node/mod.rs +++ b/cli/node/mod.rs @@ -26,11 +26,11 @@ use deno_runtime::deno_node::package_imports_resolve; use deno_runtime::deno_node::package_resolve; use deno_runtime::deno_node::NodeModuleKind; use deno_runtime::deno_node::PackageJson; +use deno_runtime::deno_node::PathClean; use deno_runtime::deno_node::RequireNpmResolver; use deno_runtime::deno_node::DEFAULT_CONDITIONS; use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME; use once_cell::sync::Lazy; -use path_clean::PathClean; use regex::Regex; use crate::file_fetcher::FileFetcher; @@ -433,9 +433,8 @@ pub fn node_resolve_npm_reference( reference: &NpmPackageReference, npm_resolver: &NpmPackageResolver, ) -> Result, AnyError> { - let package_folder = npm_resolver - .resolve_package_from_deno_module(&reference.req)? - .folder_path; + let package_folder = + npm_resolver.resolve_package_folder_from_deno_module(&reference.req)?; let resolved_path = package_config_resolve( &reference .sub_path @@ -462,15 +461,28 @@ pub fn node_resolve_binary_export( bin_name: Option<&str>, npm_resolver: &NpmPackageResolver, ) -> Result { - let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?; - let package_folder = pkg.folder_path; + fn get_package_display_name(package_json: &PackageJson) -> String { + package_json + .name + .as_ref() + .and_then(|name| { + package_json + .version + .as_ref() + .map(|version| format!("{}@{}", name, version)) + }) + .unwrap_or_else(|| format!("{}", package_json.path.display())) + } + + let package_folder = + npm_resolver.resolve_package_folder_from_deno_module(pkg_req)?; let package_json_path = package_folder.join("package.json"); let package_json = PackageJson::load(npm_resolver, package_json_path)?; let bin = match &package_json.bin { Some(bin) => bin, None => bail!( "package {} did not have a 'bin' property in its package.json", - pkg.id + get_package_display_name(&package_json), ), }; let bin_entry = match bin { @@ -490,13 +502,13 @@ pub fn node_resolve_binary_export( o.get(&pkg_req.name) } }, - _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", pkg.id), + _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", get_package_display_name(&package_json)), }; let bin_entry = match bin_entry { Some(e) => e, None => bail!( "package {} did not have a 'bin' entry for {} in its package.json", - pkg.id, + get_package_display_name(&package_json), bin_name.unwrap_or(&pkg_req.name), ), }; @@ -504,7 +516,7 @@ pub fn node_resolve_binary_export( Value::String(s) => s, _ => bail!( "package {} had a non-string sub property of 'bin' in its package.json", - pkg.id + get_package_display_name(&package_json), ), }; diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 4bde3b3406..87fa9922f5 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -2,6 +2,7 @@ use std::cmp::Ordering; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::VecDeque; use deno_ast::ModuleSpecifier; @@ -185,6 +186,26 @@ impl NpmResolutionSnapshot { } } + pub fn top_level_packages(&self) -> Vec { + self + .package_reqs + .iter() + .map(|(req, version)| NpmPackageId { + name: req.name.clone(), + version: version.clone(), + }) + .collect::>() + .into_iter() + .collect::>() + } + + pub fn package_from_id( + &self, + id: &NpmPackageId, + ) -> Option<&NpmResolutionPackage> { + self.packages.get(id) + } + pub fn resolve_package_from_package( &self, name: &str, @@ -471,8 +492,6 @@ impl NpmResolution { !self.snapshot.read().packages.is_empty() } - // todo(dsherret): for use in the lsp - #[allow(dead_code)] pub fn snapshot(&self) -> NpmResolutionSnapshot { self.snapshot.read().clone() } diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index f0231859a0..cc590e2ad6 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -1,3 +1,6 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; @@ -9,34 +12,25 @@ use deno_core::futures::future::BoxFuture; use deno_core::url::Url; use crate::npm::NpmCache; -use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; -/// Information about the local npm package. -pub struct LocalNpmPackageInfo { - /// Unique identifier. - pub id: NpmPackageId, - /// Local folder path of the npm package. - pub folder_path: PathBuf, -} - pub trait InnerNpmPackageResolver: Send + Sync { - fn resolve_package_from_deno_module( + fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result; + ) -> Result; - fn resolve_package_from_package( + fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result; + ) -> Result; - fn resolve_package_from_specifier( + fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result; + ) -> Result; fn has_packages(&self) -> bool; @@ -87,3 +81,33 @@ pub async fn cache_packages( } Ok(()) } + +pub fn ensure_registry_read_permission( + registry_path: &Path, + path: &Path, +) -> Result<(), AnyError> { + // allow reading if it's in the node_modules + if path.starts_with(®istry_path) + && path + .components() + .all(|c| !matches!(c, std::path::Component::ParentDir)) + { + // todo(dsherret): cache this? + if let Ok(registry_path) = std::fs::canonicalize(registry_path) { + match std::fs::canonicalize(path) { + Ok(path) if path.starts_with(registry_path) => { + return Ok(()); + } + Err(e) if e.kind() == ErrorKind::NotFound => { + return Ok(()); + } + _ => {} // ignore + } + } + } + + Err(deno_core::error::custom_error( + "PermissionDenied", + format!("Reading {} is not allowed", path.display()), + )) +} diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 259d9b9a04..94b963898a 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -1,7 +1,9 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::io::ErrorKind; +//! Code for global npm cache resolution. + use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use deno_ast::ModuleSpecifier; @@ -17,9 +19,10 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmRegistryApi; +use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; -use super::common::LocalNpmPackageInfo; +/// Resolves packages from the global npm cache. #[derive(Debug, Clone)] pub struct GlobalNpmPackageResolver { cache: NpmCache, @@ -39,45 +42,42 @@ impl GlobalNpmPackageResolver { } } - fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo { - LocalNpmPackageInfo { - folder_path: self.cache.package_folder(id, &self.registry_url), - id: id.clone(), - } + fn package_folder(&self, id: &NpmPackageId) -> PathBuf { + self.cache.package_folder(id, &self.registry_url) } } impl InnerNpmPackageResolver for GlobalNpmPackageResolver { - fn resolve_package_from_deno_module( + fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result { + ) -> Result { let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?; - Ok(self.local_package_info(&pkg.id)) + Ok(self.package_folder(&pkg.id)) } - fn resolve_package_from_package( + fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result { + ) -> Result { let referrer_pkg_id = self .cache .resolve_package_id_from_specifier(referrer, &self.registry_url)?; let pkg = self .resolution .resolve_package_from_package(name, &referrer_pkg_id)?; - Ok(self.local_package_info(&pkg.id)) + Ok(self.package_folder(&pkg.id)) } - fn resolve_package_from_specifier( + fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result { + ) -> Result { let pkg_id = self .cache .resolve_package_id_from_specifier(specifier, &self.registry_url)?; - Ok(self.local_package_info(&pkg_id)) + Ok(self.package_folder(&pkg_id)) } fn has_packages(&self) -> bool { @@ -103,36 +103,6 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { let registry_path = self.cache.registry_folder(&self.registry_url); - ensure_read_permission(®istry_path, path) + ensure_registry_read_permission(®istry_path, path) } } - -fn ensure_read_permission( - registry_path: &Path, - path: &Path, -) -> Result<(), AnyError> { - // allow reading if it's in the deno_dir node modules - if path.starts_with(®istry_path) - && path - .components() - .all(|c| !matches!(c, std::path::Component::ParentDir)) - { - // todo(dsherret): cache this? - if let Ok(registry_path) = std::fs::canonicalize(registry_path) { - match std::fs::canonicalize(path) { - Ok(path) if path.starts_with(registry_path) => { - return Ok(()); - } - Err(e) if e.kind() == ErrorKind::NotFound => { - return Ok(()); - } - _ => {} // ignore - } - } - } - - Err(deno_core::error::custom_error( - "PermissionDenied", - format!("Reading {} is not allowed", path.display()), - )) -} diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs new file mode 100644 index 0000000000..d92ffb84d9 --- /dev/null +++ b/cli/npm/resolvers/local.rs @@ -0,0 +1,321 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +//! Code for local node_modules resolution. + +use std::collections::HashSet; +use std::collections::VecDeque; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::FutureExt; +use deno_core::url::Url; + +use crate::fs_util; +use crate::npm::resolution::NpmResolution; +use crate::npm::resolution::NpmResolutionSnapshot; +use crate::npm::NpmCache; +use crate::npm::NpmPackageId; +use crate::npm::NpmPackageReq; +use crate::npm::NpmRegistryApi; + +use super::common::cache_packages; +use super::common::ensure_registry_read_permission; +use super::common::InnerNpmPackageResolver; + +/// Resolver that creates a local node_modules directory +/// and resolves packages from it. +#[derive(Debug, Clone)] +pub struct LocalNpmPackageResolver { + cache: NpmCache, + resolution: Arc, + registry_url: Url, + root_node_modules_path: PathBuf, + root_node_modules_specifier: ModuleSpecifier, +} + +impl LocalNpmPackageResolver { + pub fn new( + cache: NpmCache, + api: NpmRegistryApi, + node_modules_folder: PathBuf, + ) -> Self { + let registry_url = api.base_url().to_owned(); + let resolution = Arc::new(NpmResolution::new(api)); + + Self { + cache, + resolution, + registry_url, + root_node_modules_specifier: ModuleSpecifier::from_directory_path( + &node_modules_folder, + ) + .unwrap(), + root_node_modules_path: node_modules_folder, + } + } + + fn resolve_package_root(&self, path: &Path) -> PathBuf { + let mut last_found = path; + loop { + let parent = last_found.parent().unwrap(); + if parent.file_name().unwrap() == "node_modules" { + return last_found.to_path_buf(); + } else { + last_found = parent; + } + } + } + + fn resolve_folder_for_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + match self.maybe_resolve_folder_for_specifier(specifier) { + Some(path) => Ok(path), + None => bail!("could not find npm package for '{}'", specifier), + } + } + + fn maybe_resolve_folder_for_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Option { + let relative_url = + self.root_node_modules_specifier.make_relative(specifier)?; + if relative_url.starts_with("../") { + return None; + } + // it's within the directory, so use it + specifier.to_file_path().ok() + } +} + +impl InnerNpmPackageResolver for LocalNpmPackageResolver { + fn resolve_package_folder_from_deno_module( + &self, + pkg_req: &NpmPackageReq, + ) -> Result { + let resolved_package = + self.resolution.resolve_package_from_deno_module(pkg_req)?; + + // it might be at the full path if there are duplicate names + let fully_resolved_folder_path = join_package_name( + &self.root_node_modules_path, + &resolved_package.id.to_string(), + ); + Ok(if fully_resolved_folder_path.exists() { + fully_resolved_folder_path + } else { + join_package_name(&self.root_node_modules_path, &resolved_package.id.name) + }) + } + + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + ) -> Result { + let local_path = self.resolve_folder_for_specifier(referrer)?; + let package_root_path = self.resolve_package_root(&local_path); + let mut current_folder = package_root_path.as_path(); + loop { + current_folder = get_next_node_modules_ancestor(current_folder); + let sub_dir = join_package_name(current_folder, name); + if sub_dir.is_dir() { + return Ok(sub_dir); + } + if current_folder == self.root_node_modules_path { + bail!( + "could not find package '{}' from referrer '{}'.", + name, + referrer + ); + } + } + } + + fn resolve_package_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result { + let local_path = self.resolve_folder_for_specifier(specifier)?; + let package_root_path = self.resolve_package_root(&local_path); + Ok(package_root_path) + } + + fn has_packages(&self) -> bool { + self.resolution.has_packages() + } + + fn add_package_reqs( + &self, + packages: Vec, + ) -> BoxFuture<'static, Result<(), AnyError>> { + let resolver = self.clone(); + async move { + resolver.resolution.add_package_reqs(packages).await?; + cache_packages( + resolver.resolution.all_packages(), + &resolver.cache, + &resolver.registry_url, + ) + .await?; + + sync_resolution_with_fs( + &resolver.resolution.snapshot(), + &resolver.cache, + &resolver.registry_url, + &resolver.root_node_modules_path, + )?; + + Ok(()) + } + .boxed() + } + + fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { + ensure_registry_read_permission(&self.root_node_modules_path, path) + } +} + +/// Creates a pnpm style folder structure. +fn sync_resolution_with_fs( + snapshot: &NpmResolutionSnapshot, + cache: &NpmCache, + registry_url: &Url, + root_node_modules_dir_path: &Path, +) -> Result<(), AnyError> { + fn get_package_folder_name(package_id: &NpmPackageId) -> String { + package_id.to_string().replace('/', "+") + } + + let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); + fs::create_dir_all(&deno_local_registry_dir).with_context(|| { + format!("Creating '{}'", deno_local_registry_dir.display()) + })?; + + // 1. Write all the packages out the .deno directory. + // + // Copy (hardlink in future) // to + // node_modules/.deno//node_modules/ + let all_packages = snapshot.all_packages(); + for package in &all_packages { + let folder_name = get_package_folder_name(&package.id); + let folder_path = deno_local_registry_dir.join(&folder_name); + let initialized_file = folder_path.join("deno_initialized"); + if !initialized_file.exists() { + let sub_node_modules = folder_path.join("node_modules"); + let package_path = join_package_name(&sub_node_modules, &package.id.name); + fs::create_dir_all(&package_path) + .with_context(|| format!("Creating '{}'", folder_path.display()))?; + let cache_folder = cache.package_folder(&package.id, registry_url); + // for now copy, but in the future consider hard linking + fs_util::copy_dir_recursive(&cache_folder, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + // 2. Symlink all the dependencies into the .deno directory. + // + // Symlink node_modules/.deno//node_modules/ to + // node_modules/.deno//node_modules/ + for package in &all_packages { + let sub_node_modules = deno_local_registry_dir + .join(&get_package_folder_name(&package.id)) + .join("node_modules"); + for (name, dep_id) in &package.dependencies { + let dep_folder_name = get_package_folder_name(dep_id); + let dep_folder_path = join_package_name( + &deno_local_registry_dir + .join(dep_folder_name) + .join("node_modules"), + &dep_id.name, + ); + symlink_package_dir( + &dep_folder_path, + &join_package_name(&sub_node_modules, name), + )?; + } + } + + // 3. Create all the packages in the node_modules folder, which are symlinks. + // + // Symlink node_modules/ to + // node_modules/.deno//node_modules/ + let mut found_names = HashSet::new(); + let mut pending_packages = VecDeque::new(); + pending_packages.extend( + snapshot + .top_level_packages() + .into_iter() + .map(|id| (id, true)), + ); + while let Some((package_id, is_top_level)) = pending_packages.pop_front() { + let root_folder_name = if found_names.insert(package_id.name.clone()) { + package_id.name.clone() + } else if is_top_level { + package_id.to_string() + } else { + continue; // skip, already handled + }; + let local_registry_package_path = deno_local_registry_dir + .join(&get_package_folder_name(&package_id)) + .join("node_modules") + .join(&package_id.name); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(root_node_modules_dir_path, &root_folder_name), + )?; + if let Some(package) = snapshot.package_from_id(&package_id) { + for id in package.dependencies.values() { + pending_packages.push_back((id.clone(), false)); + } + } + } + + Ok(()) +} + +fn symlink_package_dir( + old_path: &Path, + new_path: &Path, +) -> Result<(), AnyError> { + let new_parent = new_path.parent().unwrap(); + if new_parent.file_name().unwrap() != "node_modules" { + // create the parent folder that will contain the symlink + fs::create_dir_all(new_parent) + .with_context(|| format!("Creating '{}'", new_parent.display()))?; + } + + // need to delete the previous symlink before creating a new one + let _ignore = fs::remove_dir_all(new_path); + fs_util::symlink_dir(old_path, new_path) +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} + +fn get_next_node_modules_ancestor(mut path: &Path) -> &Path { + loop { + path = path.parent().unwrap(); + let file_name = path.file_name().unwrap().to_string_lossy(); + if file_name == "node_modules" { + return path; + } + } +} diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 02e5be983d..3a40340f0b 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -2,9 +2,13 @@ mod common; mod global; +mod local; +use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::error::custom_error; +use deno_core::error::AnyError; +use deno_runtime::deno_node::PathClean; use deno_runtime::deno_node::RequireNpmResolver; use global::GlobalNpmPackageResolver; @@ -12,16 +16,14 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_core::error::AnyError; +use crate::fs_util; use self::common::InnerNpmPackageResolver; +use self::local::LocalNpmPackageResolver; use super::NpmCache; use super::NpmPackageReq; use super::NpmRegistryApi; -pub use self::common::LocalNpmPackageInfo; - #[derive(Clone)] pub struct NpmPackageResolver { unstable: bool, @@ -35,10 +37,17 @@ impl NpmPackageResolver { api: NpmRegistryApi, unstable: bool, no_npm: bool, + local_node_modules_path: Option, ) -> Self { - // For now, always create a GlobalNpmPackageResolver, but in the future - // this might be a local node_modules folder - let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api)); + let inner: Arc = match local_node_modules_path + { + Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( + cache, + api, + node_modules_folder, + )), + None => Arc::new(GlobalNpmPackageResolver::new(cache, api)), + }; Self { unstable, no_npm, @@ -46,36 +55,51 @@ impl NpmPackageResolver { } } - /// Resolves an npm package from a Deno module. - pub fn resolve_package_from_deno_module( + /// Resolves an npm package folder path from a Deno module. + pub fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result { - self.inner.resolve_package_from_deno_module(pkg_req) + ) -> Result { + let path = self + .inner + .resolve_package_folder_from_deno_module(pkg_req)?; + let path = fs_util::canonicalize_path_maybe_not_exists(&path)?; + log::debug!("Resolved {} to {}", pkg_req, path.display()); + Ok(path) } - /// Resolves an npm package from an npm package referrer. - pub fn resolve_package_from_package( + /// Resolves an npm package folder path from an npm package referrer. + pub fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result { - self.inner.resolve_package_from_package(name, referrer) + ) -> Result { + let path = self + .inner + .resolve_package_folder_from_package(name, referrer)?; + log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); + Ok(path) } /// Resolve the root folder of the package the provided specifier is in. /// /// This will error when the provided specifier is not in an npm package. - pub fn resolve_package_from_specifier( + pub fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result { - self.inner.resolve_package_from_specifier(specifier) + ) -> Result { + let path = self + .inner + .resolve_package_folder_from_specifier(specifier)?; + log::debug!("Resolved {} to {}", specifier, path.display()); + Ok(path) } /// Gets if the provided specifier is in an npm package. pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.resolve_package_from_specifier(specifier).is_ok() + self + .resolve_package_folder_from_specifier(specifier) + .is_ok() } /// If the resolver has resolved any npm packages. @@ -121,28 +145,27 @@ impl RequireNpmResolver for NpmPackageResolver { specifier: &str, referrer: &std::path::Path, ) -> Result { - let referrer = specifier_to_path(referrer)?; - self - .resolve_package_from_package(specifier, &referrer) - .map(|p| p.folder_path) + let referrer = path_to_specifier(referrer)?; + self.resolve_package_folder_from_package(specifier, &referrer) } fn resolve_package_folder_from_path( &self, path: &Path, ) -> Result { - let specifier = specifier_to_path(path)?; - self - .resolve_package_from_specifier(&specifier) - .map(|p| p.folder_path) + let specifier = path_to_specifier(path)?; + self.resolve_package_folder_from_specifier(&specifier) } fn in_npm_package(&self, path: &Path) -> bool { - let specifier = match ModuleSpecifier::from_file_path(path) { - Ok(p) => p, - Err(_) => return false, - }; - self.resolve_package_from_specifier(&specifier).is_ok() + let specifier = + match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) { + Ok(p) => p, + Err(_) => return false, + }; + self + .resolve_package_folder_from_specifier(&specifier) + .is_ok() } fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { @@ -150,8 +173,8 @@ impl RequireNpmResolver for NpmPackageResolver { } } -fn specifier_to_path(path: &Path) -> Result { - match ModuleSpecifier::from_file_path(&path) { +fn path_to_specifier(path: &Path) -> Result { + match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) { Ok(specifier) => Ok(specifier), Err(()) => bail!("Could not convert '{}' to url.", path.display()), } diff --git a/cli/proc_state.rs b/cli/proc_state.rs index d3c6f2cdd2..27cd0603d4 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -241,6 +241,9 @@ impl ProcState { // don't do the unstable error when in the lsp || matches!(cli_options.sub_command(), DenoSubcommand::Lsp), cli_options.no_npm(), + cli_options + .resolve_local_node_modules_folder() + .with_context(|| "Resolving local node_modules folder.")?, ); let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into(); @@ -513,15 +516,7 @@ impl ProcState { &self.npm_resolver, )) .with_context(|| { - format!( - "Could not resolve '{}' from '{}'.", - specifier, - self - .npm_resolver - .resolve_package_from_specifier(&referrer) - .unwrap() - .id - ) + format!("Could not resolve '{}' from '{}'.", specifier, referrer) }); } diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 2dcd9824f5..507bddb30c 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -534,6 +534,52 @@ itest!(builtin_module_module { http_server: true, }); +itest!(node_modules_dir_require_added_node_modules_folder { + args: + "run --unstable --node-modules-dir -A --quiet $TESTDATA/npm/require_added_nm_folder/main.js", + output: "npm/require_added_nm_folder/main.out", + envs: env_vars(), + http_server: true, + exit_code: 0, + temp_cwd: true, +}); + +itest!(node_modules_dir_with_deps { + args: "run --allow-read --allow-env --unstable --node-modules-dir $TESTDATA/npm/cjs_with_deps/main.js", + output: "npm/cjs_with_deps/main.out", + envs: env_vars(), + http_server: true, + temp_cwd: true, +}); + +#[test] +fn node_modules_dir_cache() { + let _server = http_server(); + + let deno_dir = util::new_deno_dir(); + + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(deno_dir.path()) + .arg("cache") + .arg("--unstable") + .arg("--node-modules-dir") + .arg("--quiet") + .arg(util::testdata_path().join("npm/dual_cjs_esm/main.ts")) + .envs(env_vars()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert!(output.status.success()); + + let node_modules = deno_dir.path().join("node_modules"); + assert!(node_modules + .join( + ".deno/@denotest+dual-cjs-esm@1.0.0/node_modules/@denotest/dual-cjs-esm" + ) + .exists()); + assert!(node_modules.join("@denotest/dual-cjs-esm").exists()); +} + #[test] fn ensure_registry_files_local() { // ensures the registry files all point at local tarballs diff --git a/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/index.js new file mode 100644 index 0000000000..8c8c4a0fa0 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/index.js @@ -0,0 +1,3 @@ +exports.getValue = () => { + return require(".other-package").get(); +}; diff --git a/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/package.json new file mode 100644 index 0000000000..718f1eb8cb --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "@denotest/require-added-nm-folder", + "version": "1.0.0" +} diff --git a/cli/tests/testdata/npm/require_added_nm_folder/main.js b/cli/tests/testdata/npm/require_added_nm_folder/main.js new file mode 100644 index 0000000000..723b2023aa --- /dev/null +++ b/cli/tests/testdata/npm/require_added_nm_folder/main.js @@ -0,0 +1,10 @@ +import { getValue } from "npm:@denotest/require-added-nm-folder"; + +Deno.mkdirSync("./node_modules/.other-package"); +Deno.writeTextFileSync("./node_modules/.other-package/package.json", "{}"); +Deno.writeTextFileSync( + "./node_modules/.other-package/index.js", + "exports.get = () => 5;", +); + +console.log(getValue()); diff --git a/cli/tests/testdata/npm/require_added_nm_folder/main.out b/cli/tests/testdata/npm/require_added_nm_folder/main.out new file mode 100644 index 0000000000..7ed6ff82de --- /dev/null +++ b/cli/tests/testdata/npm/require_added_nm_folder/main.out @@ -0,0 +1 @@ +5 diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index 3a2bed1053..35170b6b9c 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -270,6 +270,7 @@ pub fn compile_to_runtime_flags( import_map_path: flags.import_map_path.clone(), inspect_brk: None, inspect: None, + node_modules_dir: false, location: flags.location.clone(), lock_write: false, lock: None, diff --git a/ext/node/lib.rs b/ext/node/lib.rs index 42348915ed..5178d81f7c 100644 --- a/ext/node/lib.rs +++ b/ext/node/lib.rs @@ -15,9 +15,11 @@ use std::rc::Rc; pub mod errors; mod package_json; +mod path; mod resolution; pub use package_json::PackageJson; +pub use path::PathClean; pub use resolution::get_closest_package_json; pub use resolution::get_package_scope_config; pub use resolution::legacy_main_resolve; diff --git a/ext/node/package_json.rs b/ext/node/package_json.rs index 95d773aeb3..81daa7ca71 100644 --- a/ext/node/package_json.rs +++ b/ext/node/package_json.rs @@ -22,6 +22,7 @@ pub struct PackageJson { main: Option, // use .main(...) module: Option, // use .main(...) pub name: Option, + pub version: Option, pub path: PathBuf, pub typ: String, pub types: Option, @@ -37,6 +38,7 @@ impl PackageJson { main: None, module: None, name: None, + version: None, path, typ: "none".to_string(), types: None, @@ -71,6 +73,7 @@ impl PackageJson { let main_val = package_json.get("main"); let module_val = package_json.get("module"); let name_val = package_json.get("name"); + let version_val = package_json.get("version"); let type_val = package_json.get("type"); let bin = package_json.get("bin").map(ToOwned::to_owned); let exports = package_json.get("exports").map(|exports| { @@ -88,6 +91,7 @@ impl PackageJson { .map(|imp| imp.to_owned()); let main = main_val.and_then(|s| s.as_str()).map(|s| s.to_string()); let name = name_val.and_then(|s| s.as_str()).map(|s| s.to_string()); + let version = version_val.and_then(|s| s.as_str()).map(|s| s.to_string()); let module = module_val.and_then(|s| s.as_str()).map(|s| s.to_string()); // Ignore unknown types for forwards compatibility @@ -116,6 +120,7 @@ impl PackageJson { path, main, name, + version, module, typ, types, diff --git a/ext/node/path.rs b/ext/node/path.rs new file mode 100644 index 0000000000..8477fe7137 --- /dev/null +++ b/ext/node/path.rs @@ -0,0 +1,38 @@ +use std::path::Component; +use std::path::PathBuf; + +/// Extenion to path_clean::PathClean +pub trait PathClean { + fn clean(&self) -> T; +} + +impl PathClean for PathBuf { + fn clean(&self) -> PathBuf { + let path = path_clean::PathClean::clean(self); + if cfg!(windows) && path.to_string_lossy().contains("..\\") { + // temporary workaround because path_clean::PathClean::clean is + // not good enough on windows + let mut components = Vec::new(); + + for component in path.components() { + match component { + Component::CurDir => { + // skip + } + Component::ParentDir => { + let poped_component = components.pop(); + if !matches!(poped_component, Some(Component::Normal(_))) { + panic!("Error normalizing: {}", path.display()); + } + } + Component::Normal(_) | Component::RootDir | Component::Prefix(_) => { + components.push(component); + } + } + } + components.into_iter().collect::() + } else { + path + } + } +} diff --git a/ext/node/resolution.rs b/ext/node/resolution.rs index 52ed06116e..1bde997093 100644 --- a/ext/node/resolution.rs +++ b/ext/node/resolution.rs @@ -10,11 +10,11 @@ use deno_core::serde_json::Map; use deno_core::serde_json::Value; use deno_core::url::Url; use deno_core::ModuleSpecifier; -use path_clean::PathClean; use regex::Regex; use crate::errors; use crate::package_json::PackageJson; +use crate::path::PathClean; use crate::RequireNpmResolver; pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"]; diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index 5a357ee7de..73fe3ff9b0 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -1861,6 +1861,7 @@ pub struct CheckOutputIntegrationTest<'a> { pub http_server: bool, pub envs: Vec<(String, String)>, pub env_clear: bool, + pub temp_cwd: bool, } impl<'a> CheckOutputIntegrationTest<'a> { @@ -1874,6 +1875,11 @@ impl<'a> CheckOutputIntegrationTest<'a> { ); std::borrow::Cow::Borrowed(&self.args_vec) }; + let testdata_dir = testdata_path(); + let args = args + .iter() + .map(|arg| arg.replace("$TESTDATA", &testdata_dir.to_string_lossy())) + .collect::>(); let deno_exe = deno_exe_path(); println!("deno_exe path {}", deno_exe.display()); @@ -1884,17 +1890,21 @@ impl<'a> CheckOutputIntegrationTest<'a> { }; let (mut reader, writer) = pipe().unwrap(); - let testdata_dir = testdata_path(); let deno_dir = new_deno_dir(); // keep this alive for the test let mut command = deno_cmd_with_deno_dir(&deno_dir); - println!("deno_exe args {}", self.args); - println!("deno_exe testdata path {:?}", &testdata_dir); + let cwd = if self.temp_cwd { + deno_dir.path() + } else { + testdata_dir.as_path() + }; + println!("deno_exe args {}", args.join(" ")); + println!("deno_exe cwd {:?}", &testdata_dir); command.args(args.iter()); if self.env_clear { command.env_clear(); } command.envs(self.envs.clone()); - command.current_dir(&testdata_dir); + command.current_dir(&cwd); command.stdin(Stdio::piped()); let writer_clone = writer.try_clone().unwrap(); command.stderr(writer_clone); @@ -1949,7 +1959,7 @@ impl<'a> CheckOutputIntegrationTest<'a> { // deno test's output capturing flushes with a zero-width space in order to // synchronize the output pipes. Occassionally this zero width space // might end up in the output so strip it from the output comparison here. - if args.first() == Some(&"test") { + if args.first().map(|s| s.as_str()) == Some("test") { actual = actual.replace('\u{200B}', ""); }