1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00

feat(npm): add flag for creating and resolving npm packages to a local node_modules folder (#15971)

This commit is contained in:
David Sherret 2022-09-22 11:17:02 -04:00 committed by GitHub
parent 9a216806d5
commit 716005a0d4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 744 additions and 126 deletions

1
Cargo.lock generated
View file

@ -830,7 +830,6 @@ dependencies = [
"notify",
"once_cell",
"os_pipe",
"path-clean",
"percent-encoding 2.2.0",
"pin-project",
"pretty_assertions",

View file

@ -85,7 +85,6 @@ monch = "=0.2.0"
notify = "=5.0.0"
once_cell = "=1.14.0"
os_pipe = "=1.0.1"
path-clean = "=0.1.0"
percent-encoding = "=2.2.0"
pin-project = "1.0.11" # don't pin because they yank crates from cargo
rand = { version = "=0.8.5", features = ["small_rng"] }

View file

@ -301,6 +301,7 @@ pub struct Flags {
pub cached_only: bool,
pub type_check_mode: TypeCheckMode,
pub config_flag: ConfigFlag,
pub node_modules_dir: bool,
pub coverage_dir: Option<String>,
pub enable_testing_features: bool,
pub ignore: Vec<PathBuf>,
@ -1734,6 +1735,7 @@ fn compile_args(app: Command) -> Command {
.arg(import_map_arg())
.arg(no_remote_arg())
.arg(no_npm_arg())
.arg(local_npm_arg())
.arg(no_config_arg())
.arg(config_arg())
.arg(no_check_arg())
@ -1749,6 +1751,7 @@ fn compile_args_without_check_args(app: Command) -> Command {
.arg(import_map_arg())
.arg(no_remote_arg())
.arg(no_npm_arg())
.arg(local_npm_arg())
.arg(config_arg())
.arg(no_config_arg())
.arg(reload_arg())
@ -2153,6 +2156,12 @@ fn no_npm_arg<'a>() -> Arg<'a> {
.help("Do not resolve npm modules")
}
fn local_npm_arg<'a>() -> Arg<'a> {
Arg::new("node-modules-dir")
.long("node-modules-dir")
.help("Creates a local node_modules folder")
}
fn unsafely_ignore_certificate_errors_arg<'a>() -> Arg<'a> {
Arg::new("unsafely-ignore-certificate-errors")
.long("unsafely-ignore-certificate-errors")
@ -2799,6 +2808,7 @@ fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
import_map_arg_parse(flags, matches);
no_remote_arg_parse(flags, matches);
no_npm_arg_parse(flags, matches);
local_npm_args_parse(flags, matches);
config_args_parse(flags, matches);
no_check_arg_parse(flags, matches);
check_arg_parse(flags, matches);
@ -2814,6 +2824,7 @@ fn compile_args_without_no_check_parse(
import_map_arg_parse(flags, matches);
no_remote_arg_parse(flags, matches);
no_npm_arg_parse(flags, matches);
local_npm_args_parse(flags, matches);
config_args_parse(flags, matches);
reload_arg_parse(flags, matches);
lock_args_parse(flags, matches);
@ -3061,6 +3072,12 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
}
}
fn local_npm_args_parse(flags: &mut Flags, matches: &ArgMatches) {
if matches.is_present("node-modules-dir") {
flags.node_modules_dir = true;
}
}
fn inspect_arg_validate(val: &str) -> Result<(), String> {
match val.parse::<SocketAddr>() {
Ok(_) => Ok(()),
@ -5034,6 +5051,22 @@ mod tests {
);
}
#[test]
fn local_npm() {
let r =
flags_from_vec(svec!["deno", "run", "--node-modules-dir", "script.ts"]);
assert_eq!(
r.unwrap(),
Flags {
subcommand: DenoSubcommand::Run(RunFlags {
script: "script.ts".to_string(),
}),
node_modules_dir: true,
..Flags::default()
}
);
}
#[test]
fn cached_only() {
let r = flags_from_vec(svec!["deno", "run", "--cached-only", "script.ts"]);

View file

@ -42,6 +42,7 @@ use crate::emit::TsConfigWithIgnoredOptions;
use crate::emit::TsTypeLib;
use crate::file_fetcher::get_root_cert_store;
use crate::file_fetcher::CacheSetting;
use crate::fs_util;
use crate::lockfile::Lockfile;
use crate::version;
@ -146,6 +147,24 @@ impl CliOptions {
self.overrides.import_map_specifier = Some(path);
}
/// Resolves the path to use for a local node_modules folder.
pub fn resolve_local_node_modules_folder(
&self,
) -> Result<Option<PathBuf>, AnyError> {
let path = if !self.flags.node_modules_dir {
return Ok(None);
} else if let Some(config_path) = self
.maybe_config_file
.as_ref()
.and_then(|c| c.specifier.to_file_path().ok())
{
config_path.parent().unwrap().join("node_modules")
} else {
std::env::current_dir()?.join("node_modules")
};
Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?))
}
pub fn resolve_root_cert_store(&self) -> Result<RootCertStore, AnyError> {
get_root_cert_store(
None,

View file

@ -5,10 +5,11 @@ use deno_core::error::{uri_error, AnyError};
pub use deno_core::normalize_path;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_crypto::rand;
use deno_runtime::deno_node::PathClean;
use std::borrow::Cow;
use std::env::current_dir;
use std::fs::OpenOptions;
use std::io::{Error, Write};
use std::io::{Error, ErrorKind, Write};
use std::path::{Path, PathBuf};
use walkdir::WalkDir;
@ -75,6 +76,35 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
return Ok(path);
}
/// Canonicalizes a path which might be non-existent by going up the
/// ancestors until it finds a directory that exists, canonicalizes
/// that path, then adds back the remaining path components.
///
/// Note: When using this, you should be aware that a symlink may
/// subsequently be created along this path by some other code.
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
let path = path.to_path_buf().clean();
let mut path = path.as_path();
let mut names_stack = Vec::new();
loop {
match canonicalize_path(path) {
Ok(mut canonicalized_path) => {
for name in names_stack.into_iter().rev() {
canonicalized_path = canonicalized_path.join(name);
}
return Ok(canonicalized_path);
}
Err(err) if err.kind() == ErrorKind::NotFound => {
names_stack.push(path.file_name().unwrap());
path = path.parent().unwrap();
}
Err(err) => return Err(err),
}
}
}
#[cfg(windows)]
fn strip_unc_prefix(path: PathBuf) -> PathBuf {
use std::path::Component;
@ -294,6 +324,60 @@ pub async fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
}
}
/// Copies a directory to another directory.
///
/// Note: Does not handle symlinks.
pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
std::fs::create_dir_all(&to)
.with_context(|| format!("Creating {}", to.display()))?;
let read_dir = std::fs::read_dir(&from)
.with_context(|| format!("Reading {}", from.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let new_from = from.join(entry.file_name());
let new_to = to.join(entry.file_name());
if file_type.is_dir() {
copy_dir_recursive(&new_from, &new_to).with_context(|| {
format!("Dir {} to {}", new_from.display(), new_to.display())
})?;
} else if file_type.is_file() {
std::fs::copy(&new_from, &new_to).with_context(|| {
format!("Copying {} to {}", new_from.display(), new_to.display())
})?;
}
}
Ok(())
}
pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> {
let err_mapper = |err: Error| {
Error::new(
err.kind(),
format!(
"{}, symlink '{}' -> '{}'",
err,
oldpath.display(),
newpath.display()
),
)
};
#[cfg(unix)]
{
use std::os::unix::fs::symlink;
symlink(&oldpath, &newpath).map_err(err_mapper)?;
}
#[cfg(not(unix))]
{
use std::os::windows::fs::symlink_dir;
symlink_dir(&oldpath, &newpath).map_err(err_mapper)?;
}
Ok(())
}
/// Attempts to convert a specifier to a file path. By default, uses the Url
/// crate's `to_file_path()` method, but falls back to try and resolve unix-style
/// paths on Windows.

View file

@ -26,11 +26,11 @@ use deno_runtime::deno_node::package_imports_resolve;
use deno_runtime::deno_node::package_resolve;
use deno_runtime::deno_node::NodeModuleKind;
use deno_runtime::deno_node::PackageJson;
use deno_runtime::deno_node::PathClean;
use deno_runtime::deno_node::RequireNpmResolver;
use deno_runtime::deno_node::DEFAULT_CONDITIONS;
use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME;
use once_cell::sync::Lazy;
use path_clean::PathClean;
use regex::Regex;
use crate::file_fetcher::FileFetcher;
@ -433,9 +433,8 @@ pub fn node_resolve_npm_reference(
reference: &NpmPackageReference,
npm_resolver: &NpmPackageResolver,
) -> Result<Option<NodeResolution>, AnyError> {
let package_folder = npm_resolver
.resolve_package_from_deno_module(&reference.req)?
.folder_path;
let package_folder =
npm_resolver.resolve_package_folder_from_deno_module(&reference.req)?;
let resolved_path = package_config_resolve(
&reference
.sub_path
@ -462,15 +461,28 @@ pub fn node_resolve_binary_export(
bin_name: Option<&str>,
npm_resolver: &NpmPackageResolver,
) -> Result<NodeResolution, AnyError> {
let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?;
let package_folder = pkg.folder_path;
fn get_package_display_name(package_json: &PackageJson) -> String {
package_json
.name
.as_ref()
.and_then(|name| {
package_json
.version
.as_ref()
.map(|version| format!("{}@{}", name, version))
})
.unwrap_or_else(|| format!("{}", package_json.path.display()))
}
let package_folder =
npm_resolver.resolve_package_folder_from_deno_module(pkg_req)?;
let package_json_path = package_folder.join("package.json");
let package_json = PackageJson::load(npm_resolver, package_json_path)?;
let bin = match &package_json.bin {
Some(bin) => bin,
None => bail!(
"package {} did not have a 'bin' property in its package.json",
pkg.id
get_package_display_name(&package_json),
),
};
let bin_entry = match bin {
@ -490,13 +502,13 @@ pub fn node_resolve_binary_export(
o.get(&pkg_req.name)
}
},
_ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", pkg.id),
_ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", get_package_display_name(&package_json)),
};
let bin_entry = match bin_entry {
Some(e) => e,
None => bail!(
"package {} did not have a 'bin' entry for {} in its package.json",
pkg.id,
get_package_display_name(&package_json),
bin_name.unwrap_or(&pkg_req.name),
),
};
@ -504,7 +516,7 @@ pub fn node_resolve_binary_export(
Value::String(s) => s,
_ => bail!(
"package {} had a non-string sub property of 'bin' in its package.json",
pkg.id
get_package_display_name(&package_json),
),
};

View file

@ -2,6 +2,7 @@
use std::cmp::Ordering;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::VecDeque;
use deno_ast::ModuleSpecifier;
@ -185,6 +186,26 @@ impl NpmResolutionSnapshot {
}
}
pub fn top_level_packages(&self) -> Vec<NpmPackageId> {
self
.package_reqs
.iter()
.map(|(req, version)| NpmPackageId {
name: req.name.clone(),
version: version.clone(),
})
.collect::<HashSet<_>>()
.into_iter()
.collect::<Vec<_>>()
}
pub fn package_from_id(
&self,
id: &NpmPackageId,
) -> Option<&NpmResolutionPackage> {
self.packages.get(id)
}
pub fn resolve_package_from_package(
&self,
name: &str,
@ -471,8 +492,6 @@ impl NpmResolution {
!self.snapshot.read().packages.is_empty()
}
// todo(dsherret): for use in the lsp
#[allow(dead_code)]
pub fn snapshot(&self) -> NpmResolutionSnapshot {
self.snapshot.read().clone()
}

View file

@ -1,3 +1,6 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
@ -9,34 +12,25 @@ use deno_core::futures::future::BoxFuture;
use deno_core::url::Url;
use crate::npm::NpmCache;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmResolutionPackage;
/// Information about the local npm package.
pub struct LocalNpmPackageInfo {
/// Unique identifier.
pub id: NpmPackageId,
/// Local folder path of the npm package.
pub folder_path: PathBuf,
}
pub trait InnerNpmPackageResolver: Send + Sync {
fn resolve_package_from_deno_module(
fn resolve_package_folder_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError>;
) -> Result<PathBuf, AnyError>;
fn resolve_package_from_package(
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
) -> Result<PathBuf, AnyError>;
fn resolve_package_from_specifier(
fn resolve_package_folder_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
) -> Result<PathBuf, AnyError>;
fn has_packages(&self) -> bool;
@ -87,3 +81,33 @@ pub async fn cache_packages(
}
Ok(())
}
pub fn ensure_registry_read_permission(
registry_path: &Path,
path: &Path,
) -> Result<(), AnyError> {
// allow reading if it's in the node_modules
if path.starts_with(&registry_path)
&& path
.components()
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}
Err(e) if e.kind() == ErrorKind::NotFound => {
return Ok(());
}
_ => {} // ignore
}
}
}
Err(deno_core::error::custom_error(
"PermissionDenied",
format!("Reading {} is not allowed", path.display()),
))
}

View file

@ -1,7 +1,9 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use std::io::ErrorKind;
//! Code for global npm cache resolution.
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
@ -17,9 +19,10 @@ use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmRegistryApi;
use super::common::ensure_registry_read_permission;
use super::common::InnerNpmPackageResolver;
use super::common::LocalNpmPackageInfo;
/// Resolves packages from the global npm cache.
#[derive(Debug, Clone)]
pub struct GlobalNpmPackageResolver {
cache: NpmCache,
@ -39,45 +42,42 @@ impl GlobalNpmPackageResolver {
}
}
fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo {
LocalNpmPackageInfo {
folder_path: self.cache.package_folder(id, &self.registry_url),
id: id.clone(),
}
fn package_folder(&self, id: &NpmPackageId) -> PathBuf {
self.cache.package_folder(id, &self.registry_url)
}
}
impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
fn resolve_package_from_deno_module(
fn resolve_package_folder_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
) -> Result<PathBuf, AnyError> {
let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?;
Ok(self.local_package_info(&pkg.id))
Ok(self.package_folder(&pkg.id))
}
fn resolve_package_from_package(
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
) -> Result<PathBuf, AnyError> {
let referrer_pkg_id = self
.cache
.resolve_package_id_from_specifier(referrer, &self.registry_url)?;
let pkg = self
.resolution
.resolve_package_from_package(name, &referrer_pkg_id)?;
Ok(self.local_package_info(&pkg.id))
Ok(self.package_folder(&pkg.id))
}
fn resolve_package_from_specifier(
fn resolve_package_folder_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
) -> Result<PathBuf, AnyError> {
let pkg_id = self
.cache
.resolve_package_id_from_specifier(specifier, &self.registry_url)?;
Ok(self.local_package_info(&pkg_id))
Ok(self.package_folder(&pkg_id))
}
fn has_packages(&self) -> bool {
@ -103,36 +103,6 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_read_permission(&registry_path, path)
ensure_registry_read_permission(&registry_path, path)
}
}
fn ensure_read_permission(
registry_path: &Path,
path: &Path,
) -> Result<(), AnyError> {
// allow reading if it's in the deno_dir node modules
if path.starts_with(&registry_path)
&& path
.components()
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}
Err(e) if e.kind() == ErrorKind::NotFound => {
return Ok(());
}
_ => {} // ignore
}
}
}
Err(deno_core::error::custom_error(
"PermissionDenied",
format!("Reading {} is not allowed", path.display()),
))
}

321
cli/npm/resolvers/local.rs Normal file
View file

@ -0,0 +1,321 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
//! Code for local node_modules resolution.
use std::collections::HashSet;
use std::collections::VecDeque;
use std::fs;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::FutureExt;
use deno_core::url::Url;
use crate::fs_util;
use crate::npm::resolution::NpmResolution;
use crate::npm::resolution::NpmResolutionSnapshot;
use crate::npm::NpmCache;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmRegistryApi;
use super::common::cache_packages;
use super::common::ensure_registry_read_permission;
use super::common::InnerNpmPackageResolver;
/// Resolver that creates a local node_modules directory
/// and resolves packages from it.
#[derive(Debug, Clone)]
pub struct LocalNpmPackageResolver {
cache: NpmCache,
resolution: Arc<NpmResolution>,
registry_url: Url,
root_node_modules_path: PathBuf,
root_node_modules_specifier: ModuleSpecifier,
}
impl LocalNpmPackageResolver {
pub fn new(
cache: NpmCache,
api: NpmRegistryApi,
node_modules_folder: PathBuf,
) -> Self {
let registry_url = api.base_url().to_owned();
let resolution = Arc::new(NpmResolution::new(api));
Self {
cache,
resolution,
registry_url,
root_node_modules_specifier: ModuleSpecifier::from_directory_path(
&node_modules_folder,
)
.unwrap(),
root_node_modules_path: node_modules_folder,
}
}
fn resolve_package_root(&self, path: &Path) -> PathBuf {
let mut last_found = path;
loop {
let parent = last_found.parent().unwrap();
if parent.file_name().unwrap() == "node_modules" {
return last_found.to_path_buf();
} else {
last_found = parent;
}
}
}
fn resolve_folder_for_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<PathBuf, AnyError> {
match self.maybe_resolve_folder_for_specifier(specifier) {
Some(path) => Ok(path),
None => bail!("could not find npm package for '{}'", specifier),
}
}
fn maybe_resolve_folder_for_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Option<PathBuf> {
let relative_url =
self.root_node_modules_specifier.make_relative(specifier)?;
if relative_url.starts_with("../") {
return None;
}
// it's within the directory, so use it
specifier.to_file_path().ok()
}
}
impl InnerNpmPackageResolver for LocalNpmPackageResolver {
fn resolve_package_folder_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<PathBuf, AnyError> {
let resolved_package =
self.resolution.resolve_package_from_deno_module(pkg_req)?;
// it might be at the full path if there are duplicate names
let fully_resolved_folder_path = join_package_name(
&self.root_node_modules_path,
&resolved_package.id.to_string(),
);
Ok(if fully_resolved_folder_path.exists() {
fully_resolved_folder_path
} else {
join_package_name(&self.root_node_modules_path, &resolved_package.id.name)
})
}
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<PathBuf, AnyError> {
let local_path = self.resolve_folder_for_specifier(referrer)?;
let package_root_path = self.resolve_package_root(&local_path);
let mut current_folder = package_root_path.as_path();
loop {
current_folder = get_next_node_modules_ancestor(current_folder);
let sub_dir = join_package_name(current_folder, name);
if sub_dir.is_dir() {
return Ok(sub_dir);
}
if current_folder == self.root_node_modules_path {
bail!(
"could not find package '{}' from referrer '{}'.",
name,
referrer
);
}
}
}
fn resolve_package_folder_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<PathBuf, AnyError> {
let local_path = self.resolve_folder_for_specifier(specifier)?;
let package_root_path = self.resolve_package_root(&local_path);
Ok(package_root_path)
}
fn has_packages(&self) -> bool {
self.resolution.has_packages()
}
fn add_package_reqs(
&self,
packages: Vec<NpmPackageReq>,
) -> BoxFuture<'static, Result<(), AnyError>> {
let resolver = self.clone();
async move {
resolver.resolution.add_package_reqs(packages).await?;
cache_packages(
resolver.resolution.all_packages(),
&resolver.cache,
&resolver.registry_url,
)
.await?;
sync_resolution_with_fs(
&resolver.resolution.snapshot(),
&resolver.cache,
&resolver.registry_url,
&resolver.root_node_modules_path,
)?;
Ok(())
}
.boxed()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
ensure_registry_read_permission(&self.root_node_modules_path, path)
}
}
/// Creates a pnpm style folder structure.
fn sync_resolution_with_fs(
snapshot: &NpmResolutionSnapshot,
cache: &NpmCache,
registry_url: &Url,
root_node_modules_dir_path: &Path,
) -> Result<(), AnyError> {
fn get_package_folder_name(package_id: &NpmPackageId) -> String {
package_id.to_string().replace('/', "+")
}
let deno_local_registry_dir = root_node_modules_dir_path.join(".deno");
fs::create_dir_all(&deno_local_registry_dir).with_context(|| {
format!("Creating '{}'", deno_local_registry_dir.display())
})?;
// 1. Write all the packages out the .deno directory.
//
// Copy (hardlink in future) <global_registry_cache>/<package_id>/ to
// node_modules/.deno/<package_id>/node_modules/<package_name>
let all_packages = snapshot.all_packages();
for package in &all_packages {
let folder_name = get_package_folder_name(&package.id);
let folder_path = deno_local_registry_dir.join(&folder_name);
let initialized_file = folder_path.join("deno_initialized");
if !initialized_file.exists() {
let sub_node_modules = folder_path.join("node_modules");
let package_path = join_package_name(&sub_node_modules, &package.id.name);
fs::create_dir_all(&package_path)
.with_context(|| format!("Creating '{}'", folder_path.display()))?;
let cache_folder = cache.package_folder(&package.id, registry_url);
// for now copy, but in the future consider hard linking
fs_util::copy_dir_recursive(&cache_folder, &package_path)?;
// write out a file that indicates this folder has been initialized
fs::write(initialized_file, "")?;
}
}
// 2. Symlink all the dependencies into the .deno directory.
//
// Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to
// node_modules/.deno/<dep_id>/node_modules/<dep_package_name>
for package in &all_packages {
let sub_node_modules = deno_local_registry_dir
.join(&get_package_folder_name(&package.id))
.join("node_modules");
for (name, dep_id) in &package.dependencies {
let dep_folder_name = get_package_folder_name(dep_id);
let dep_folder_path = join_package_name(
&deno_local_registry_dir
.join(dep_folder_name)
.join("node_modules"),
&dep_id.name,
);
symlink_package_dir(
&dep_folder_path,
&join_package_name(&sub_node_modules, name),
)?;
}
}
// 3. Create all the packages in the node_modules folder, which are symlinks.
//
// Symlink node_modules/<package_name> to
// node_modules/.deno/<package_id>/node_modules/<package_name>
let mut found_names = HashSet::new();
let mut pending_packages = VecDeque::new();
pending_packages.extend(
snapshot
.top_level_packages()
.into_iter()
.map(|id| (id, true)),
);
while let Some((package_id, is_top_level)) = pending_packages.pop_front() {
let root_folder_name = if found_names.insert(package_id.name.clone()) {
package_id.name.clone()
} else if is_top_level {
package_id.to_string()
} else {
continue; // skip, already handled
};
let local_registry_package_path = deno_local_registry_dir
.join(&get_package_folder_name(&package_id))
.join("node_modules")
.join(&package_id.name);
symlink_package_dir(
&local_registry_package_path,
&join_package_name(root_node_modules_dir_path, &root_folder_name),
)?;
if let Some(package) = snapshot.package_from_id(&package_id) {
for id in package.dependencies.values() {
pending_packages.push_back((id.clone(), false));
}
}
}
Ok(())
}
fn symlink_package_dir(
old_path: &Path,
new_path: &Path,
) -> Result<(), AnyError> {
let new_parent = new_path.parent().unwrap();
if new_parent.file_name().unwrap() != "node_modules" {
// create the parent folder that will contain the symlink
fs::create_dir_all(new_parent)
.with_context(|| format!("Creating '{}'", new_parent.display()))?;
}
// need to delete the previous symlink before creating a new one
let _ignore = fs::remove_dir_all(new_path);
fs_util::symlink_dir(old_path, new_path)
}
fn join_package_name(path: &Path, package_name: &str) -> PathBuf {
let mut path = path.to_path_buf();
// ensure backslashes are used on windows
for part in package_name.split('/') {
path = path.join(part);
}
path
}
fn get_next_node_modules_ancestor(mut path: &Path) -> &Path {
loop {
path = path.parent().unwrap();
let file_name = path.file_name().unwrap().to_string_lossy();
if file_name == "node_modules" {
return path;
}
}
}

View file

@ -2,9 +2,13 @@
mod common;
mod global;
mod local;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_runtime::deno_node::PathClean;
use deno_runtime::deno_node::RequireNpmResolver;
use global::GlobalNpmPackageResolver;
@ -12,16 +16,14 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use crate::fs_util;
use self::common::InnerNpmPackageResolver;
use self::local::LocalNpmPackageResolver;
use super::NpmCache;
use super::NpmPackageReq;
use super::NpmRegistryApi;
pub use self::common::LocalNpmPackageInfo;
#[derive(Clone)]
pub struct NpmPackageResolver {
unstable: bool,
@ -35,10 +37,17 @@ impl NpmPackageResolver {
api: NpmRegistryApi,
unstable: bool,
no_npm: bool,
local_node_modules_path: Option<PathBuf>,
) -> Self {
// For now, always create a GlobalNpmPackageResolver, but in the future
// this might be a local node_modules folder
let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api));
let inner: Arc<dyn InnerNpmPackageResolver> = match local_node_modules_path
{
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
cache,
api,
node_modules_folder,
)),
None => Arc::new(GlobalNpmPackageResolver::new(cache, api)),
};
Self {
unstable,
no_npm,
@ -46,36 +55,51 @@ impl NpmPackageResolver {
}
}
/// Resolves an npm package from a Deno module.
pub fn resolve_package_from_deno_module(
/// Resolves an npm package folder path from a Deno module.
pub fn resolve_package_folder_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_deno_module(pkg_req)
) -> Result<PathBuf, AnyError> {
let path = self
.inner
.resolve_package_folder_from_deno_module(pkg_req)?;
let path = fs_util::canonicalize_path_maybe_not_exists(&path)?;
log::debug!("Resolved {} to {}", pkg_req, path.display());
Ok(path)
}
/// Resolves an npm package from an npm package referrer.
pub fn resolve_package_from_package(
/// Resolves an npm package folder path from an npm package referrer.
pub fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_package(name, referrer)
) -> Result<PathBuf, AnyError> {
let path = self
.inner
.resolve_package_folder_from_package(name, referrer)?;
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
/// Resolve the root folder of the package the provided specifier is in.
///
/// This will error when the provided specifier is not in an npm package.
pub fn resolve_package_from_specifier(
pub fn resolve_package_folder_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_specifier(specifier)
) -> Result<PathBuf, AnyError> {
let path = self
.inner
.resolve_package_folder_from_specifier(specifier)?;
log::debug!("Resolved {} to {}", specifier, path.display());
Ok(path)
}
/// Gets if the provided specifier is in an npm package.
pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
self.resolve_package_from_specifier(specifier).is_ok()
self
.resolve_package_folder_from_specifier(specifier)
.is_ok()
}
/// If the resolver has resolved any npm packages.
@ -121,28 +145,27 @@ impl RequireNpmResolver for NpmPackageResolver {
specifier: &str,
referrer: &std::path::Path,
) -> Result<PathBuf, AnyError> {
let referrer = specifier_to_path(referrer)?;
self
.resolve_package_from_package(specifier, &referrer)
.map(|p| p.folder_path)
let referrer = path_to_specifier(referrer)?;
self.resolve_package_folder_from_package(specifier, &referrer)
}
fn resolve_package_folder_from_path(
&self,
path: &Path,
) -> Result<PathBuf, AnyError> {
let specifier = specifier_to_path(path)?;
self
.resolve_package_from_specifier(&specifier)
.map(|p| p.folder_path)
let specifier = path_to_specifier(path)?;
self.resolve_package_folder_from_specifier(&specifier)
}
fn in_npm_package(&self, path: &Path) -> bool {
let specifier = match ModuleSpecifier::from_file_path(path) {
Ok(p) => p,
Err(_) => return false,
};
self.resolve_package_from_specifier(&specifier).is_ok()
let specifier =
match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) {
Ok(p) => p,
Err(_) => return false,
};
self
.resolve_package_folder_from_specifier(&specifier)
.is_ok()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
@ -150,8 +173,8 @@ impl RequireNpmResolver for NpmPackageResolver {
}
}
fn specifier_to_path(path: &Path) -> Result<ModuleSpecifier, AnyError> {
match ModuleSpecifier::from_file_path(&path) {
fn path_to_specifier(path: &Path) -> Result<ModuleSpecifier, AnyError> {
match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) {
Ok(specifier) => Ok(specifier),
Err(()) => bail!("Could not convert '{}' to url.", path.display()),
}

View file

@ -241,6 +241,9 @@ impl ProcState {
// don't do the unstable error when in the lsp
|| matches!(cli_options.sub_command(), DenoSubcommand::Lsp),
cli_options.no_npm(),
cli_options
.resolve_local_node_modules_folder()
.with_context(|| "Resolving local node_modules folder.")?,
);
let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into();
@ -513,15 +516,7 @@ impl ProcState {
&self.npm_resolver,
))
.with_context(|| {
format!(
"Could not resolve '{}' from '{}'.",
specifier,
self
.npm_resolver
.resolve_package_from_specifier(&referrer)
.unwrap()
.id
)
format!("Could not resolve '{}' from '{}'.", specifier, referrer)
});
}

View file

@ -534,6 +534,52 @@ itest!(builtin_module_module {
http_server: true,
});
itest!(node_modules_dir_require_added_node_modules_folder {
args:
"run --unstable --node-modules-dir -A --quiet $TESTDATA/npm/require_added_nm_folder/main.js",
output: "npm/require_added_nm_folder/main.out",
envs: env_vars(),
http_server: true,
exit_code: 0,
temp_cwd: true,
});
itest!(node_modules_dir_with_deps {
args: "run --allow-read --allow-env --unstable --node-modules-dir $TESTDATA/npm/cjs_with_deps/main.js",
output: "npm/cjs_with_deps/main.out",
envs: env_vars(),
http_server: true,
temp_cwd: true,
});
#[test]
fn node_modules_dir_cache() {
let _server = http_server();
let deno_dir = util::new_deno_dir();
let deno = util::deno_cmd_with_deno_dir(&deno_dir)
.current_dir(deno_dir.path())
.arg("cache")
.arg("--unstable")
.arg("--node-modules-dir")
.arg("--quiet")
.arg(util::testdata_path().join("npm/dual_cjs_esm/main.ts"))
.envs(env_vars())
.spawn()
.unwrap();
let output = deno.wait_with_output().unwrap();
assert!(output.status.success());
let node_modules = deno_dir.path().join("node_modules");
assert!(node_modules
.join(
".deno/@denotest+dual-cjs-esm@1.0.0/node_modules/@denotest/dual-cjs-esm"
)
.exists());
assert!(node_modules.join("@denotest/dual-cjs-esm").exists());
}
#[test]
fn ensure_registry_files_local() {
// ensures the registry files all point at local tarballs

View file

@ -0,0 +1,3 @@
exports.getValue = () => {
return require(".other-package").get();
};

View file

@ -0,0 +1,4 @@
{
"name": "@denotest/require-added-nm-folder",
"version": "1.0.0"
}

View file

@ -0,0 +1,10 @@
import { getValue } from "npm:@denotest/require-added-nm-folder";
Deno.mkdirSync("./node_modules/.other-package");
Deno.writeTextFileSync("./node_modules/.other-package/package.json", "{}");
Deno.writeTextFileSync(
"./node_modules/.other-package/index.js",
"exports.get = () => 5;",
);
console.log(getValue());

View file

@ -0,0 +1 @@
5

View file

@ -270,6 +270,7 @@ pub fn compile_to_runtime_flags(
import_map_path: flags.import_map_path.clone(),
inspect_brk: None,
inspect: None,
node_modules_dir: false,
location: flags.location.clone(),
lock_write: false,
lock: None,

View file

@ -15,9 +15,11 @@ use std::rc::Rc;
pub mod errors;
mod package_json;
mod path;
mod resolution;
pub use package_json::PackageJson;
pub use path::PathClean;
pub use resolution::get_closest_package_json;
pub use resolution::get_package_scope_config;
pub use resolution::legacy_main_resolve;

View file

@ -22,6 +22,7 @@ pub struct PackageJson {
main: Option<String>, // use .main(...)
module: Option<String>, // use .main(...)
pub name: Option<String>,
pub version: Option<String>,
pub path: PathBuf,
pub typ: String,
pub types: Option<String>,
@ -37,6 +38,7 @@ impl PackageJson {
main: None,
module: None,
name: None,
version: None,
path,
typ: "none".to_string(),
types: None,
@ -71,6 +73,7 @@ impl PackageJson {
let main_val = package_json.get("main");
let module_val = package_json.get("module");
let name_val = package_json.get("name");
let version_val = package_json.get("version");
let type_val = package_json.get("type");
let bin = package_json.get("bin").map(ToOwned::to_owned);
let exports = package_json.get("exports").map(|exports| {
@ -88,6 +91,7 @@ impl PackageJson {
.map(|imp| imp.to_owned());
let main = main_val.and_then(|s| s.as_str()).map(|s| s.to_string());
let name = name_val.and_then(|s| s.as_str()).map(|s| s.to_string());
let version = version_val.and_then(|s| s.as_str()).map(|s| s.to_string());
let module = module_val.and_then(|s| s.as_str()).map(|s| s.to_string());
// Ignore unknown types for forwards compatibility
@ -116,6 +120,7 @@ impl PackageJson {
path,
main,
name,
version,
module,
typ,
types,

38
ext/node/path.rs Normal file
View file

@ -0,0 +1,38 @@
use std::path::Component;
use std::path::PathBuf;
/// Extenion to path_clean::PathClean
pub trait PathClean<T> {
fn clean(&self) -> T;
}
impl PathClean<PathBuf> for PathBuf {
fn clean(&self) -> PathBuf {
let path = path_clean::PathClean::clean(self);
if cfg!(windows) && path.to_string_lossy().contains("..\\") {
// temporary workaround because path_clean::PathClean::clean is
// not good enough on windows
let mut components = Vec::new();
for component in path.components() {
match component {
Component::CurDir => {
// skip
}
Component::ParentDir => {
let poped_component = components.pop();
if !matches!(poped_component, Some(Component::Normal(_))) {
panic!("Error normalizing: {}", path.display());
}
}
Component::Normal(_) | Component::RootDir | Component::Prefix(_) => {
components.push(component);
}
}
}
components.into_iter().collect::<PathBuf>()
} else {
path
}
}
}

View file

@ -10,11 +10,11 @@ use deno_core::serde_json::Map;
use deno_core::serde_json::Value;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use path_clean::PathClean;
use regex::Regex;
use crate::errors;
use crate::package_json::PackageJson;
use crate::path::PathClean;
use crate::RequireNpmResolver;
pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"];

View file

@ -1861,6 +1861,7 @@ pub struct CheckOutputIntegrationTest<'a> {
pub http_server: bool,
pub envs: Vec<(String, String)>,
pub env_clear: bool,
pub temp_cwd: bool,
}
impl<'a> CheckOutputIntegrationTest<'a> {
@ -1874,6 +1875,11 @@ impl<'a> CheckOutputIntegrationTest<'a> {
);
std::borrow::Cow::Borrowed(&self.args_vec)
};
let testdata_dir = testdata_path();
let args = args
.iter()
.map(|arg| arg.replace("$TESTDATA", &testdata_dir.to_string_lossy()))
.collect::<Vec<_>>();
let deno_exe = deno_exe_path();
println!("deno_exe path {}", deno_exe.display());
@ -1884,17 +1890,21 @@ impl<'a> CheckOutputIntegrationTest<'a> {
};
let (mut reader, writer) = pipe().unwrap();
let testdata_dir = testdata_path();
let deno_dir = new_deno_dir(); // keep this alive for the test
let mut command = deno_cmd_with_deno_dir(&deno_dir);
println!("deno_exe args {}", self.args);
println!("deno_exe testdata path {:?}", &testdata_dir);
let cwd = if self.temp_cwd {
deno_dir.path()
} else {
testdata_dir.as_path()
};
println!("deno_exe args {}", args.join(" "));
println!("deno_exe cwd {:?}", &testdata_dir);
command.args(args.iter());
if self.env_clear {
command.env_clear();
}
command.envs(self.envs.clone());
command.current_dir(&testdata_dir);
command.current_dir(&cwd);
command.stdin(Stdio::piped());
let writer_clone = writer.try_clone().unwrap();
command.stderr(writer_clone);
@ -1949,7 +1959,7 @@ impl<'a> CheckOutputIntegrationTest<'a> {
// deno test's output capturing flushes with a zero-width space in order to
// synchronize the output pipes. Occassionally this zero width space
// might end up in the output so strip it from the output comparison here.
if args.first() == Some(&"test") {
if args.first().map(|s| s.as_str()) == Some("test") {
actual = actual.replace('\u{200B}', "");
}