feat(npm): add flag for creating and resolving npm packages to a local node_modules folder (#15971)

Cargo.lock

@@ -830,7 +830,6 @@ dependencies = [
  "notify",
  "once_cell",
  "os_pipe",
- "path-clean",
  "percent-encoding 2.2.0",
  "pin-project",
  "pretty_assertions",

cli/Cargo.toml

@@ -85,7 +85,6 @@ monch = "=0.2.0"
 notify = "=5.0.0"
 once_cell = "=1.14.0"
 os_pipe = "=1.0.1"
-path-clean = "=0.1.0"
 percent-encoding = "=2.2.0"
 pin-project = "1.0.11" # don't pin because they yank crates from cargo
 rand = { version = "=0.8.5", features = ["small_rng"] }

cli/args/flags.rs

@@ -301,6 +301,7 @@ pub struct Flags {
   pub cached_only: bool,
   pub type_check_mode: TypeCheckMode,
   pub config_flag: ConfigFlag,
+  pub node_modules_dir: bool,
   pub coverage_dir: Option<String>,
   pub enable_testing_features: bool,
   pub ignore: Vec<PathBuf>,
@@ -1734,6 +1735,7 @@ fn compile_args(app: Command) -> Command {
     .arg(import_map_arg())
     .arg(no_remote_arg())
     .arg(no_npm_arg())
+    .arg(local_npm_arg())
     .arg(no_config_arg())
     .arg(config_arg())
     .arg(no_check_arg())
@@ -1749,6 +1751,7 @@ fn compile_args_without_check_args(app: Command) -> Command {
     .arg(import_map_arg())
     .arg(no_remote_arg())
     .arg(no_npm_arg())
+    .arg(local_npm_arg())
     .arg(config_arg())
     .arg(no_config_arg())
     .arg(reload_arg())
@@ -2153,6 +2156,12 @@ fn no_npm_arg<'a>() -> Arg<'a> {
     .help("Do not resolve npm modules")
 }
 
+fn local_npm_arg<'a>() -> Arg<'a> {
+  Arg::new("node-modules-dir")
+    .long("node-modules-dir")
+    .help("Creates a local node_modules folder")
+}
+
 fn unsafely_ignore_certificate_errors_arg<'a>() -> Arg<'a> {
   Arg::new("unsafely-ignore-certificate-errors")
     .long("unsafely-ignore-certificate-errors")
@@ -2799,6 +2808,7 @@ fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   import_map_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
   no_npm_arg_parse(flags, matches);
+  local_npm_args_parse(flags, matches);
   config_args_parse(flags, matches);
   no_check_arg_parse(flags, matches);
   check_arg_parse(flags, matches);
@@ -2814,6 +2824,7 @@ fn compile_args_without_no_check_parse(
   import_map_arg_parse(flags, matches);
   no_remote_arg_parse(flags, matches);
   no_npm_arg_parse(flags, matches);
+  local_npm_args_parse(flags, matches);
   config_args_parse(flags, matches);
   reload_arg_parse(flags, matches);
   lock_args_parse(flags, matches);
@@ -3061,6 +3072,12 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   }
 }
 
+fn local_npm_args_parse(flags: &mut Flags, matches: &ArgMatches) {
+  if matches.is_present("node-modules-dir") {
+    flags.node_modules_dir = true;
+  }
+}
+
 fn inspect_arg_validate(val: &str) -> Result<(), String> {
   match val.parse::<SocketAddr>() {
     Ok(_) => Ok(()),
@@ -5034,6 +5051,22 @@ mod tests {
     );
   }
 
+  #[test]
+  fn local_npm() {
+    let r =
+      flags_from_vec(svec!["deno", "run", "--node-modules-dir", "script.ts"]);
+    assert_eq!(
+      r.unwrap(),
+      Flags {
+        subcommand: DenoSubcommand::Run(RunFlags {
+          script: "script.ts".to_string(),
+        }),
+        node_modules_dir: true,
+        ..Flags::default()
+      }
+    );
+  }
+
   #[test]
   fn cached_only() {
     let r = flags_from_vec(svec!["deno", "run", "--cached-only", "script.ts"]);

cli/args/mod.rs

@@ -42,6 +42,7 @@ use crate::emit::TsConfigWithIgnoredOptions;
 use crate::emit::TsTypeLib;
 use crate::file_fetcher::get_root_cert_store;
 use crate::file_fetcher::CacheSetting;
+use crate::fs_util;
 use crate::lockfile::Lockfile;
 use crate::version;
 
@@ -146,6 +147,24 @@ impl CliOptions {
     self.overrides.import_map_specifier = Some(path);
   }
 
+  /// Resolves the path to use for a local node_modules folder.
+  pub fn resolve_local_node_modules_folder(
+    &self,
+  ) -> Result<Option<PathBuf>, AnyError> {
+    let path = if !self.flags.node_modules_dir {
+      return Ok(None);
+    } else if let Some(config_path) = self
+      .maybe_config_file
+      .as_ref()
+      .and_then(|c| c.specifier.to_file_path().ok())
+    {
+      config_path.parent().unwrap().join("node_modules")
+    } else {
+      std::env::current_dir()?.join("node_modules")
+    };
+    Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?))
+  }
+
   pub fn resolve_root_cert_store(&self) -> Result<RootCertStore, AnyError> {
     get_root_cert_store(
       None,

cli/fs_util.rs

@@ -5,10 +5,11 @@ use deno_core::error::{uri_error, AnyError};
 pub use deno_core::normalize_path;
 use deno_core::ModuleSpecifier;
 use deno_runtime::deno_crypto::rand;
+use deno_runtime::deno_node::PathClean;
 use std::borrow::Cow;
 use std::env::current_dir;
 use std::fs::OpenOptions;
-use std::io::{Error, Write};
+use std::io::{Error, ErrorKind, Write};
 use std::path::{Path, PathBuf};
 use walkdir::WalkDir;
 
@@ -75,6 +76,35 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
   return Ok(path);
 }
 
+/// Canonicalizes a path which might be non-existent by going up the
+/// ancestors until it finds a directory that exists, canonicalizes
+/// that path, then adds back the remaining path components.
+///
+/// Note: When using this, you should be aware that a symlink may
+/// subsequently be created along this path by some other code.
+pub fn canonicalize_path_maybe_not_exists(
+  path: &Path,
+) -> Result<PathBuf, Error> {
+  let path = path.to_path_buf().clean();
+  let mut path = path.as_path();
+  let mut names_stack = Vec::new();
+  loop {
+    match canonicalize_path(path) {
+      Ok(mut canonicalized_path) => {
+        for name in names_stack.into_iter().rev() {
+          canonicalized_path = canonicalized_path.join(name);
+        }
+        return Ok(canonicalized_path);
+      }
+      Err(err) if err.kind() == ErrorKind::NotFound => {
+        names_stack.push(path.file_name().unwrap());
+        path = path.parent().unwrap();
+      }
+      Err(err) => return Err(err),
+    }
+  }
+}
+
 #[cfg(windows)]
 fn strip_unc_prefix(path: PathBuf) -> PathBuf {
   use std::path::Component;
@@ -294,6 +324,60 @@ pub async fn remove_dir_all_if_exists(path: &Path) -> std::io::Result<()> {
   }
 }
 
+/// Copies a directory to another directory.
+///
+/// Note: Does not handle symlinks.
+pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
+  std::fs::create_dir_all(&to)
+    .with_context(|| format!("Creating {}", to.display()))?;
+  let read_dir = std::fs::read_dir(&from)
+    .with_context(|| format!("Reading {}", from.display()))?;
+
+  for entry in read_dir {
+    let entry = entry?;
+    let file_type = entry.file_type()?;
+    let new_from = from.join(entry.file_name());
+    let new_to = to.join(entry.file_name());
+
+    if file_type.is_dir() {
+      copy_dir_recursive(&new_from, &new_to).with_context(|| {
+        format!("Dir {} to {}", new_from.display(), new_to.display())
+      })?;
+    } else if file_type.is_file() {
+      std::fs::copy(&new_from, &new_to).with_context(|| {
+        format!("Copying {} to {}", new_from.display(), new_to.display())
+      })?;
+    }
+  }
+
+  Ok(())
+}
+
+pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> {
+  let err_mapper = |err: Error| {
+    Error::new(
+      err.kind(),
+      format!(
+        "{}, symlink '{}' -> '{}'",
+        err,
+        oldpath.display(),
+        newpath.display()
+      ),
+    )
+  };
+  #[cfg(unix)]
+  {
+    use std::os::unix::fs::symlink;
+    symlink(&oldpath, &newpath).map_err(err_mapper)?;
+  }
+  #[cfg(not(unix))]
+  {
+    use std::os::windows::fs::symlink_dir;
+    symlink_dir(&oldpath, &newpath).map_err(err_mapper)?;
+  }
+  Ok(())
+}
+
 /// Attempts to convert a specifier to a file path. By default, uses the Url
 /// crate's `to_file_path()` method, but falls back to try and resolve unix-style
 /// paths on Windows.

cli/node/mod.rs

@@ -26,11 +26,11 @@ use deno_runtime::deno_node::package_imports_resolve;
 use deno_runtime::deno_node::package_resolve;
 use deno_runtime::deno_node::NodeModuleKind;
 use deno_runtime::deno_node::PackageJson;
+use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use deno_runtime::deno_node::DEFAULT_CONDITIONS;
 use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME;
 use once_cell::sync::Lazy;
-use path_clean::PathClean;
 use regex::Regex;
 
 use crate::file_fetcher::FileFetcher;
@@ -433,9 +433,8 @@ pub fn node_resolve_npm_reference(
   reference: &NpmPackageReference,
   npm_resolver: &NpmPackageResolver,
 ) -> Result<Option<NodeResolution>, AnyError> {
-  let package_folder = npm_resolver
+  let package_folder =
-    .resolve_package_from_deno_module(&reference.req)?
+    npm_resolver.resolve_package_folder_from_deno_module(&reference.req)?;
-    .folder_path;
   let resolved_path = package_config_resolve(
     &reference
       .sub_path
@@ -462,15 +461,28 @@ pub fn node_resolve_binary_export(
   bin_name: Option<&str>,
   npm_resolver: &NpmPackageResolver,
 ) -> Result<NodeResolution, AnyError> {
-  let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?;
+  fn get_package_display_name(package_json: &PackageJson) -> String {
-  let package_folder = pkg.folder_path;
+    package_json
+      .name
+      .as_ref()
+      .and_then(|name| {
+        package_json
+          .version
+          .as_ref()
+          .map(|version| format!("{}@{}", name, version))
+      })
+      .unwrap_or_else(|| format!("{}", package_json.path.display()))
+  }
+
+  let package_folder =
+    npm_resolver.resolve_package_folder_from_deno_module(pkg_req)?;
   let package_json_path = package_folder.join("package.json");
   let package_json = PackageJson::load(npm_resolver, package_json_path)?;
   let bin = match &package_json.bin {
     Some(bin) => bin,
     None => bail!(
       "package {} did not have a 'bin' property in its package.json",
-      pkg.id
+      get_package_display_name(&package_json),
     ),
   };
   let bin_entry = match bin {
@@ -490,13 +502,13 @@ pub fn node_resolve_binary_export(
         o.get(&pkg_req.name)
       }
     },
-    _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", pkg.id),
+    _ => bail!("package {} did not have a 'bin' property with a string or object value in its package.json", get_package_display_name(&package_json)),
   };
   let bin_entry = match bin_entry {
     Some(e) => e,
     None => bail!(
       "package {} did not have a 'bin' entry for {} in its package.json",
-      pkg.id,
+      get_package_display_name(&package_json),
       bin_name.unwrap_or(&pkg_req.name),
     ),
   };
@@ -504,7 +516,7 @@ pub fn node_resolve_binary_export(
     Value::String(s) => s,
     _ => bail!(
       "package {} had a non-string sub property of 'bin' in its package.json",
-      pkg.id
+      get_package_display_name(&package_json),
     ),
   };
 

cli/npm/resolution.rs

@@ -2,6 +2,7 @@
 
 use std::cmp::Ordering;
 use std::collections::HashMap;
+use std::collections::HashSet;
 use std::collections::VecDeque;
 
 use deno_ast::ModuleSpecifier;
@@ -185,6 +186,26 @@ impl NpmResolutionSnapshot {
     }
   }
 
+  pub fn top_level_packages(&self) -> Vec<NpmPackageId> {
+    self
+      .package_reqs
+      .iter()
+      .map(|(req, version)| NpmPackageId {
+        name: req.name.clone(),
+        version: version.clone(),
+      })
+      .collect::<HashSet<_>>()
+      .into_iter()
+      .collect::<Vec<_>>()
+  }
+
+  pub fn package_from_id(
+    &self,
+    id: &NpmPackageId,
+  ) -> Option<&NpmResolutionPackage> {
+    self.packages.get(id)
+  }
+
   pub fn resolve_package_from_package(
     &self,
     name: &str,
@@ -471,8 +492,6 @@ impl NpmResolution {
     !self.snapshot.read().packages.is_empty()
   }
 
-  // todo(dsherret): for use in the lsp
-  #[allow(dead_code)]
   pub fn snapshot(&self) -> NpmResolutionSnapshot {
     self.snapshot.read().clone()
   }

cli/npm/resolvers/common.rs

@@ -1,3 +1,6 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::io::ErrorKind;
 use std::path::Path;
 use std::path::PathBuf;
 
@@ -9,34 +12,25 @@ use deno_core::futures::future::BoxFuture;
 use deno_core::url::Url;
 
 use crate::npm::NpmCache;
-use crate::npm::NpmPackageId;
 use crate::npm::NpmPackageReq;
 use crate::npm::NpmResolutionPackage;
 
-/// Information about the local npm package.
-pub struct LocalNpmPackageInfo {
-  /// Unique identifier.
-  pub id: NpmPackageId,
-  /// Local folder path of the npm package.
-  pub folder_path: PathBuf,
-}
-
 pub trait InnerNpmPackageResolver: Send + Sync {
-  fn resolve_package_from_deno_module(
+  fn resolve_package_folder_from_deno_module(
     &self,
     pkg_req: &NpmPackageReq,
-  ) -> Result<LocalNpmPackageInfo, AnyError>;
+  ) -> Result<PathBuf, AnyError>;
 
-  fn resolve_package_from_package(
+  fn resolve_package_folder_from_package(
     &self,
     name: &str,
     referrer: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError>;
+  ) -> Result<PathBuf, AnyError>;
 
-  fn resolve_package_from_specifier(
+  fn resolve_package_folder_from_specifier(
     &self,
     specifier: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError>;
+  ) -> Result<PathBuf, AnyError>;
 
   fn has_packages(&self) -> bool;
 
@@ -87,3 +81,33 @@ pub async fn cache_packages(
   }
   Ok(())
 }
+
+pub fn ensure_registry_read_permission(
+  registry_path: &Path,
+  path: &Path,
+) -> Result<(), AnyError> {
+  // allow reading if it's in the node_modules
+  if path.starts_with(&registry_path)
+    && path
+      .components()
+      .all(|c| !matches!(c, std::path::Component::ParentDir))
+  {
+    // todo(dsherret): cache this?
+    if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
+      match std::fs::canonicalize(path) {
+        Ok(path) if path.starts_with(registry_path) => {
+          return Ok(());
+        }
+        Err(e) if e.kind() == ErrorKind::NotFound => {
+          return Ok(());
+        }
+        _ => {} // ignore
+      }
+    }
+  }
+
+  Err(deno_core::error::custom_error(
+    "PermissionDenied",
+    format!("Reading {} is not allowed", path.display()),
+  ))
+}

cli/npm/resolvers/global.rs

@@ -1,7 +1,9 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 
-use std::io::ErrorKind;
+//! Code for global npm cache resolution.
+
 use std::path::Path;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use deno_ast::ModuleSpecifier;
@@ -17,9 +19,10 @@ use crate::npm::NpmPackageId;
 use crate::npm::NpmPackageReq;
 use crate::npm::NpmRegistryApi;
 
+use super::common::ensure_registry_read_permission;
 use super::common::InnerNpmPackageResolver;
-use super::common::LocalNpmPackageInfo;
 
+/// Resolves packages from the global npm cache.
 #[derive(Debug, Clone)]
 pub struct GlobalNpmPackageResolver {
   cache: NpmCache,
@@ -39,45 +42,42 @@ impl GlobalNpmPackageResolver {
     }
   }
 
-  fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo {
+  fn package_folder(&self, id: &NpmPackageId) -> PathBuf {
-    LocalNpmPackageInfo {
+    self.cache.package_folder(id, &self.registry_url)
-      folder_path: self.cache.package_folder(id, &self.registry_url),
-      id: id.clone(),
-    }
   }
 }
 
 impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
-  fn resolve_package_from_deno_module(
+  fn resolve_package_folder_from_deno_module(
     &self,
     pkg_req: &NpmPackageReq,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
     let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?;
-    Ok(self.local_package_info(&pkg.id))
+    Ok(self.package_folder(&pkg.id))
   }
 
-  fn resolve_package_from_package(
+  fn resolve_package_folder_from_package(
     &self,
     name: &str,
     referrer: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
     let referrer_pkg_id = self
       .cache
       .resolve_package_id_from_specifier(referrer, &self.registry_url)?;
     let pkg = self
       .resolution
       .resolve_package_from_package(name, &referrer_pkg_id)?;
-    Ok(self.local_package_info(&pkg.id))
+    Ok(self.package_folder(&pkg.id))
   }
 
-  fn resolve_package_from_specifier(
+  fn resolve_package_folder_from_specifier(
     &self,
     specifier: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
     let pkg_id = self
       .cache
       .resolve_package_id_from_specifier(specifier, &self.registry_url)?;
-    Ok(self.local_package_info(&pkg_id))
+    Ok(self.package_folder(&pkg_id))
   }
 
   fn has_packages(&self) -> bool {
@@ -103,36 +103,6 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
 
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
     let registry_path = self.cache.registry_folder(&self.registry_url);
-    ensure_read_permission(&registry_path, path)
+    ensure_registry_read_permission(&registry_path, path)
   }
 }
-
-fn ensure_read_permission(
-  registry_path: &Path,
-  path: &Path,
-) -> Result<(), AnyError> {
-  // allow reading if it's in the deno_dir node modules
-  if path.starts_with(&registry_path)
-    && path
-      .components()
-      .all(|c| !matches!(c, std::path::Component::ParentDir))
-  {
-    // todo(dsherret): cache this?
-    if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
-      match std::fs::canonicalize(path) {
-        Ok(path) if path.starts_with(registry_path) => {
-          return Ok(());
-        }
-        Err(e) if e.kind() == ErrorKind::NotFound => {
-          return Ok(());
-        }
-        _ => {} // ignore
-      }
-    }
-  }
-
-  Err(deno_core::error::custom_error(
-    "PermissionDenied",
-    format!("Reading {} is not allowed", path.display()),
-  ))
-}

cli/npm/resolvers/local.rs

@@ -0,0 +1,321 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+//! Code for local node_modules resolution.
+
+use std::collections::HashSet;
+use std::collections::VecDeque;
+use std::fs;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::futures::future::BoxFuture;
+use deno_core::futures::FutureExt;
+use deno_core::url::Url;
+
+use crate::fs_util;
+use crate::npm::resolution::NpmResolution;
+use crate::npm::resolution::NpmResolutionSnapshot;
+use crate::npm::NpmCache;
+use crate::npm::NpmPackageId;
+use crate::npm::NpmPackageReq;
+use crate::npm::NpmRegistryApi;
+
+use super::common::cache_packages;
+use super::common::ensure_registry_read_permission;
+use super::common::InnerNpmPackageResolver;
+
+/// Resolver that creates a local node_modules directory
+/// and resolves packages from it.
+#[derive(Debug, Clone)]
+pub struct LocalNpmPackageResolver {
+  cache: NpmCache,
+  resolution: Arc<NpmResolution>,
+  registry_url: Url,
+  root_node_modules_path: PathBuf,
+  root_node_modules_specifier: ModuleSpecifier,
+}
+
+impl LocalNpmPackageResolver {
+  pub fn new(
+    cache: NpmCache,
+    api: NpmRegistryApi,
+    node_modules_folder: PathBuf,
+  ) -> Self {
+    let registry_url = api.base_url().to_owned();
+    let resolution = Arc::new(NpmResolution::new(api));
+
+    Self {
+      cache,
+      resolution,
+      registry_url,
+      root_node_modules_specifier: ModuleSpecifier::from_directory_path(
+        &node_modules_folder,
+      )
+      .unwrap(),
+      root_node_modules_path: node_modules_folder,
+    }
+  }
+
+  fn resolve_package_root(&self, path: &Path) -> PathBuf {
+    let mut last_found = path;
+    loop {
+      let parent = last_found.parent().unwrap();
+      if parent.file_name().unwrap() == "node_modules" {
+        return last_found.to_path_buf();
+      } else {
+        last_found = parent;
+      }
+    }
+  }
+
+  fn resolve_folder_for_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    match self.maybe_resolve_folder_for_specifier(specifier) {
+      Some(path) => Ok(path),
+      None => bail!("could not find npm package for '{}'", specifier),
+    }
+  }
+
+  fn maybe_resolve_folder_for_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Option<PathBuf> {
+    let relative_url =
+      self.root_node_modules_specifier.make_relative(specifier)?;
+    if relative_url.starts_with("../") {
+      return None;
+    }
+    // it's within the directory, so use it
+    specifier.to_file_path().ok()
+  }
+}
+
+impl InnerNpmPackageResolver for LocalNpmPackageResolver {
+  fn resolve_package_folder_from_deno_module(
+    &self,
+    pkg_req: &NpmPackageReq,
+  ) -> Result<PathBuf, AnyError> {
+    let resolved_package =
+      self.resolution.resolve_package_from_deno_module(pkg_req)?;
+
+    // it might be at the full path if there are duplicate names
+    let fully_resolved_folder_path = join_package_name(
+      &self.root_node_modules_path,
+      &resolved_package.id.to_string(),
+    );
+    Ok(if fully_resolved_folder_path.exists() {
+      fully_resolved_folder_path
+    } else {
+      join_package_name(&self.root_node_modules_path, &resolved_package.id.name)
+    })
+  }
+
+  fn resolve_package_folder_from_package(
+    &self,
+    name: &str,
+    referrer: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    let local_path = self.resolve_folder_for_specifier(referrer)?;
+    let package_root_path = self.resolve_package_root(&local_path);
+    let mut current_folder = package_root_path.as_path();
+    loop {
+      current_folder = get_next_node_modules_ancestor(current_folder);
+      let sub_dir = join_package_name(current_folder, name);
+      if sub_dir.is_dir() {
+        return Ok(sub_dir);
+      }
+      if current_folder == self.root_node_modules_path {
+        bail!(
+          "could not find package '{}' from referrer '{}'.",
+          name,
+          referrer
+        );
+      }
+    }
+  }
+
+  fn resolve_package_folder_from_specifier(
+    &self,
+    specifier: &ModuleSpecifier,
+  ) -> Result<PathBuf, AnyError> {
+    let local_path = self.resolve_folder_for_specifier(specifier)?;
+    let package_root_path = self.resolve_package_root(&local_path);
+    Ok(package_root_path)
+  }
+
+  fn has_packages(&self) -> bool {
+    self.resolution.has_packages()
+  }
+
+  fn add_package_reqs(
+    &self,
+    packages: Vec<NpmPackageReq>,
+  ) -> BoxFuture<'static, Result<(), AnyError>> {
+    let resolver = self.clone();
+    async move {
+      resolver.resolution.add_package_reqs(packages).await?;
+      cache_packages(
+        resolver.resolution.all_packages(),
+        &resolver.cache,
+        &resolver.registry_url,
+      )
+      .await?;
+
+      sync_resolution_with_fs(
+        &resolver.resolution.snapshot(),
+        &resolver.cache,
+        &resolver.registry_url,
+        &resolver.root_node_modules_path,
+      )?;
+
+      Ok(())
+    }
+    .boxed()
+  }
+
+  fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
+    ensure_registry_read_permission(&self.root_node_modules_path, path)
+  }
+}
+
+/// Creates a pnpm style folder structure.
+fn sync_resolution_with_fs(
+  snapshot: &NpmResolutionSnapshot,
+  cache: &NpmCache,
+  registry_url: &Url,
+  root_node_modules_dir_path: &Path,
+) -> Result<(), AnyError> {
+  fn get_package_folder_name(package_id: &NpmPackageId) -> String {
+    package_id.to_string().replace('/', "+")
+  }
+
+  let deno_local_registry_dir = root_node_modules_dir_path.join(".deno");
+  fs::create_dir_all(&deno_local_registry_dir).with_context(|| {
+    format!("Creating '{}'", deno_local_registry_dir.display())
+  })?;
+
+  // 1. Write all the packages out the .deno directory.
+  //
+  // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to
+  // node_modules/.deno/<package_id>/node_modules/<package_name>
+  let all_packages = snapshot.all_packages();
+  for package in &all_packages {
+    let folder_name = get_package_folder_name(&package.id);
+    let folder_path = deno_local_registry_dir.join(&folder_name);
+    let initialized_file = folder_path.join("deno_initialized");
+    if !initialized_file.exists() {
+      let sub_node_modules = folder_path.join("node_modules");
+      let package_path = join_package_name(&sub_node_modules, &package.id.name);
+      fs::create_dir_all(&package_path)
+        .with_context(|| format!("Creating '{}'", folder_path.display()))?;
+      let cache_folder = cache.package_folder(&package.id, registry_url);
+      // for now copy, but in the future consider hard linking
+      fs_util::copy_dir_recursive(&cache_folder, &package_path)?;
+      // write out a file that indicates this folder has been initialized
+      fs::write(initialized_file, "")?;
+    }
+  }
+
+  // 2. Symlink all the dependencies into the .deno directory.
+  //
+  // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to
+  // node_modules/.deno/<dep_id>/node_modules/<dep_package_name>
+  for package in &all_packages {
+    let sub_node_modules = deno_local_registry_dir
+      .join(&get_package_folder_name(&package.id))
+      .join("node_modules");
+    for (name, dep_id) in &package.dependencies {
+      let dep_folder_name = get_package_folder_name(dep_id);
+      let dep_folder_path = join_package_name(
+        &deno_local_registry_dir
+          .join(dep_folder_name)
+          .join("node_modules"),
+        &dep_id.name,
+      );
+      symlink_package_dir(
+        &dep_folder_path,
+        &join_package_name(&sub_node_modules, name),
+      )?;
+    }
+  }
+
+  // 3. Create all the packages in the node_modules folder, which are symlinks.
+  //
+  // Symlink node_modules/<package_name> to
+  // node_modules/.deno/<package_id>/node_modules/<package_name>
+  let mut found_names = HashSet::new();
+  let mut pending_packages = VecDeque::new();
+  pending_packages.extend(
+    snapshot
+      .top_level_packages()
+      .into_iter()
+      .map(|id| (id, true)),
+  );
+  while let Some((package_id, is_top_level)) = pending_packages.pop_front() {
+    let root_folder_name = if found_names.insert(package_id.name.clone()) {
+      package_id.name.clone()
+    } else if is_top_level {
+      package_id.to_string()
+    } else {
+      continue; // skip, already handled
+    };
+    let local_registry_package_path = deno_local_registry_dir
+      .join(&get_package_folder_name(&package_id))
+      .join("node_modules")
+      .join(&package_id.name);
+
+    symlink_package_dir(
+      &local_registry_package_path,
+      &join_package_name(root_node_modules_dir_path, &root_folder_name),
+    )?;
+    if let Some(package) = snapshot.package_from_id(&package_id) {
+      for id in package.dependencies.values() {
+        pending_packages.push_back((id.clone(), false));
+      }
+    }
+  }
+
+  Ok(())
+}
+
+fn symlink_package_dir(
+  old_path: &Path,
+  new_path: &Path,
+) -> Result<(), AnyError> {
+  let new_parent = new_path.parent().unwrap();
+  if new_parent.file_name().unwrap() != "node_modules" {
+    // create the parent folder that will contain the symlink
+    fs::create_dir_all(new_parent)
+      .with_context(|| format!("Creating '{}'", new_parent.display()))?;
+  }
+
+  // need to delete the previous symlink before creating a new one
+  let _ignore = fs::remove_dir_all(new_path);
+  fs_util::symlink_dir(old_path, new_path)
+}
+
+fn join_package_name(path: &Path, package_name: &str) -> PathBuf {
+  let mut path = path.to_path_buf();
+  // ensure backslashes are used on windows
+  for part in package_name.split('/') {
+    path = path.join(part);
+  }
+  path
+}
+
+fn get_next_node_modules_ancestor(mut path: &Path) -> &Path {
+  loop {
+    path = path.parent().unwrap();
+    let file_name = path.file_name().unwrap().to_string_lossy();
+    if file_name == "node_modules" {
+      return path;
+    }
+  }
+}

cli/npm/resolvers/mod.rs

@@ -2,9 +2,13 @@
 
 mod common;
 mod global;
+mod local;
 
+use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::bail;
 use deno_core::error::custom_error;
+use deno_core::error::AnyError;
+use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use global::GlobalNpmPackageResolver;
 
@@ -12,16 +16,14 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-use deno_ast::ModuleSpecifier;
+use crate::fs_util;
-use deno_core::error::AnyError;
 
 use self::common::InnerNpmPackageResolver;
+use self::local::LocalNpmPackageResolver;
 use super::NpmCache;
 use super::NpmPackageReq;
 use super::NpmRegistryApi;
 
-pub use self::common::LocalNpmPackageInfo;
-
 #[derive(Clone)]
 pub struct NpmPackageResolver {
   unstable: bool,
@@ -35,10 +37,17 @@ impl NpmPackageResolver {
     api: NpmRegistryApi,
     unstable: bool,
     no_npm: bool,
+    local_node_modules_path: Option<PathBuf>,
   ) -> Self {
-    // For now, always create a GlobalNpmPackageResolver, but in the future
+    let inner: Arc<dyn InnerNpmPackageResolver> = match local_node_modules_path
-    // this might be a local node_modules folder
+    {
-    let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api));
+      Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
+        cache,
+        api,
+        node_modules_folder,
+      )),
+      None => Arc::new(GlobalNpmPackageResolver::new(cache, api)),
+    };
     Self {
       unstable,
       no_npm,
@@ -46,36 +55,51 @@ impl NpmPackageResolver {
     }
   }
 
-  /// Resolves an npm package from a Deno module.
+  /// Resolves an npm package folder path from a Deno module.
-  pub fn resolve_package_from_deno_module(
+  pub fn resolve_package_folder_from_deno_module(
     &self,
     pkg_req: &NpmPackageReq,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
-    self.inner.resolve_package_from_deno_module(pkg_req)
+    let path = self
+      .inner
+      .resolve_package_folder_from_deno_module(pkg_req)?;
+    let path = fs_util::canonicalize_path_maybe_not_exists(&path)?;
+    log::debug!("Resolved {} to {}", pkg_req, path.display());
+    Ok(path)
   }
 
-  /// Resolves an npm package from an npm package referrer.
+  /// Resolves an npm package folder path from an npm package referrer.
-  pub fn resolve_package_from_package(
+  pub fn resolve_package_folder_from_package(
     &self,
     name: &str,
     referrer: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
-    self.inner.resolve_package_from_package(name, referrer)
+    let path = self
+      .inner
+      .resolve_package_folder_from_package(name, referrer)?;
+    log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
+    Ok(path)
   }
 
   /// Resolve the root folder of the package the provided specifier is in.
   ///
   /// This will error when the provided specifier is not in an npm package.
-  pub fn resolve_package_from_specifier(
+  pub fn resolve_package_folder_from_specifier(
     &self,
     specifier: &ModuleSpecifier,
-  ) -> Result<LocalNpmPackageInfo, AnyError> {
+  ) -> Result<PathBuf, AnyError> {
-    self.inner.resolve_package_from_specifier(specifier)
+    let path = self
+      .inner
+      .resolve_package_folder_from_specifier(specifier)?;
+    log::debug!("Resolved {} to {}", specifier, path.display());
+    Ok(path)
   }
 
   /// Gets if the provided specifier is in an npm package.
   pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
-    self.resolve_package_from_specifier(specifier).is_ok()
+    self
+      .resolve_package_folder_from_specifier(specifier)
+      .is_ok()
   }
 
   /// If the resolver has resolved any npm packages.
@@ -121,28 +145,27 @@ impl RequireNpmResolver for NpmPackageResolver {
     specifier: &str,
     referrer: &std::path::Path,
   ) -> Result<PathBuf, AnyError> {
-    let referrer = specifier_to_path(referrer)?;
+    let referrer = path_to_specifier(referrer)?;
-    self
+    self.resolve_package_folder_from_package(specifier, &referrer)
-      .resolve_package_from_package(specifier, &referrer)
-      .map(|p| p.folder_path)
   }
 
   fn resolve_package_folder_from_path(
     &self,
     path: &Path,
   ) -> Result<PathBuf, AnyError> {
-    let specifier = specifier_to_path(path)?;
+    let specifier = path_to_specifier(path)?;
-    self
+    self.resolve_package_folder_from_specifier(&specifier)
-      .resolve_package_from_specifier(&specifier)
-      .map(|p| p.folder_path)
   }
 
   fn in_npm_package(&self, path: &Path) -> bool {
-    let specifier = match ModuleSpecifier::from_file_path(path) {
+    let specifier =
+      match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) {
         Ok(p) => p,
         Err(_) => return false,
       };
-    self.resolve_package_from_specifier(&specifier).is_ok()
+    self
+      .resolve_package_folder_from_specifier(&specifier)
+      .is_ok()
   }
 
   fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
@@ -150,8 +173,8 @@ impl RequireNpmResolver for NpmPackageResolver {
   }
 }
 
-fn specifier_to_path(path: &Path) -> Result<ModuleSpecifier, AnyError> {
+fn path_to_specifier(path: &Path) -> Result<ModuleSpecifier, AnyError> {
-  match ModuleSpecifier::from_file_path(&path) {
+  match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) {
     Ok(specifier) => Ok(specifier),
     Err(()) => bail!("Could not convert '{}' to url.", path.display()),
   }

cli/proc_state.rs

@@ -241,6 +241,9 @@ impl ProcState {
         // don't do the unstable error when in the lsp
         || matches!(cli_options.sub_command(), DenoSubcommand::Lsp),
       cli_options.no_npm(),
+      cli_options
+        .resolve_local_node_modules_folder()
+        .with_context(|| "Resolving local node_modules folder.")?,
     );
 
     let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into();
@@ -513,15 +516,7 @@ impl ProcState {
             &self.npm_resolver,
           ))
           .with_context(|| {
-            format!(
+            format!("Could not resolve '{}' from '{}'.", specifier, referrer)
-              "Could not resolve '{}' from '{}'.",
-              specifier,
-              self
-                .npm_resolver
-                .resolve_package_from_specifier(&referrer)
-                .unwrap()
-                .id
-            )
           });
       }
 

cli/tests/integration/npm_tests.rs

@@ -534,6 +534,52 @@ itest!(builtin_module_module {
   http_server: true,
 });
 
+itest!(node_modules_dir_require_added_node_modules_folder {
+  args:
+    "run --unstable --node-modules-dir -A --quiet $TESTDATA/npm/require_added_nm_folder/main.js",
+  output: "npm/require_added_nm_folder/main.out",
+  envs: env_vars(),
+  http_server: true,
+  exit_code: 0,
+  temp_cwd: true,
+});
+
+itest!(node_modules_dir_with_deps {
+  args: "run --allow-read --allow-env --unstable --node-modules-dir $TESTDATA/npm/cjs_with_deps/main.js",
+  output: "npm/cjs_with_deps/main.out",
+  envs: env_vars(),
+  http_server: true,
+  temp_cwd: true,
+});
+
+#[test]
+fn node_modules_dir_cache() {
+  let _server = http_server();
+
+  let deno_dir = util::new_deno_dir();
+
+  let deno = util::deno_cmd_with_deno_dir(&deno_dir)
+    .current_dir(deno_dir.path())
+    .arg("cache")
+    .arg("--unstable")
+    .arg("--node-modules-dir")
+    .arg("--quiet")
+    .arg(util::testdata_path().join("npm/dual_cjs_esm/main.ts"))
+    .envs(env_vars())
+    .spawn()
+    .unwrap();
+  let output = deno.wait_with_output().unwrap();
+  assert!(output.status.success());
+
+  let node_modules = deno_dir.path().join("node_modules");
+  assert!(node_modules
+    .join(
+      ".deno/@denotest+dual-cjs-esm@1.0.0/node_modules/@denotest/dual-cjs-esm"
+    )
+    .exists());
+  assert!(node_modules.join("@denotest/dual-cjs-esm").exists());
+}
+
 #[test]
 fn ensure_registry_files_local() {
   // ensures the registry files all point at local tarballs

cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/index.js

@@ -0,0 +1,3 @@
+exports.getValue = () => {
+  return require(".other-package").get();
+};

cli/tests/testdata/npm/registry/@denotest/require-added-nm-folder/1.0.0/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@denotest/require-added-nm-folder",
+  "version": "1.0.0"
+}

cli/tests/testdata/npm/require_added_nm_folder/main.js

@@ -0,0 +1,10 @@
+import { getValue } from "npm:@denotest/require-added-nm-folder";
+
+Deno.mkdirSync("./node_modules/.other-package");
+Deno.writeTextFileSync("./node_modules/.other-package/package.json", "{}");
+Deno.writeTextFileSync(
+  "./node_modules/.other-package/index.js",
+  "exports.get = () => 5;",
+);
+
+console.log(getValue());

cli/tests/testdata/npm/require_added_nm_folder/main.out

@@ -0,0 +1 @@
+5

cli/tools/standalone.rs

@@ -270,6 +270,7 @@ pub fn compile_to_runtime_flags(
     import_map_path: flags.import_map_path.clone(),
     inspect_brk: None,
     inspect: None,
+    node_modules_dir: false,
     location: flags.location.clone(),
     lock_write: false,
     lock: None,

ext/node/lib.rs

@@ -15,9 +15,11 @@ use std::rc::Rc;
 
 pub mod errors;
 mod package_json;
+mod path;
 mod resolution;
 
 pub use package_json::PackageJson;
+pub use path::PathClean;
 pub use resolution::get_closest_package_json;
 pub use resolution::get_package_scope_config;
 pub use resolution::legacy_main_resolve;

ext/node/package_json.rs

@@ -22,6 +22,7 @@ pub struct PackageJson {
   main: Option<String>,   // use .main(...)
   module: Option<String>, // use .main(...)
   pub name: Option<String>,
+  pub version: Option<String>,
   pub path: PathBuf,
   pub typ: String,
   pub types: Option<String>,
@@ -37,6 +38,7 @@ impl PackageJson {
       main: None,
       module: None,
       name: None,
+      version: None,
       path,
       typ: "none".to_string(),
       types: None,
@@ -71,6 +73,7 @@ impl PackageJson {
     let main_val = package_json.get("main");
     let module_val = package_json.get("module");
     let name_val = package_json.get("name");
+    let version_val = package_json.get("version");
     let type_val = package_json.get("type");
     let bin = package_json.get("bin").map(ToOwned::to_owned);
     let exports = package_json.get("exports").map(|exports| {
@@ -88,6 +91,7 @@ impl PackageJson {
       .map(|imp| imp.to_owned());
     let main = main_val.and_then(|s| s.as_str()).map(|s| s.to_string());
     let name = name_val.and_then(|s| s.as_str()).map(|s| s.to_string());
+    let version = version_val.and_then(|s| s.as_str()).map(|s| s.to_string());
     let module = module_val.and_then(|s| s.as_str()).map(|s| s.to_string());
 
     // Ignore unknown types for forwards compatibility
@@ -116,6 +120,7 @@ impl PackageJson {
       path,
       main,
       name,
+      version,
       module,
       typ,
       types,

ext/node/path.rs

@@ -0,0 +1,38 @@
+use std::path::Component;
+use std::path::PathBuf;
+
+/// Extenion to path_clean::PathClean
+pub trait PathClean<T> {
+  fn clean(&self) -> T;
+}
+
+impl PathClean<PathBuf> for PathBuf {
+  fn clean(&self) -> PathBuf {
+    let path = path_clean::PathClean::clean(self);
+    if cfg!(windows) && path.to_string_lossy().contains("..\\") {
+      // temporary workaround because path_clean::PathClean::clean is
+      // not good enough on windows
+      let mut components = Vec::new();
+
+      for component in path.components() {
+        match component {
+          Component::CurDir => {
+            // skip
+          }
+          Component::ParentDir => {
+            let poped_component = components.pop();
+            if !matches!(poped_component, Some(Component::Normal(_))) {
+              panic!("Error normalizing: {}", path.display());
+            }
+          }
+          Component::Normal(_) | Component::RootDir | Component::Prefix(_) => {
+            components.push(component);
+          }
+        }
+      }
+      components.into_iter().collect::<PathBuf>()
+    } else {
+      path
+    }
+  }
+}

ext/node/resolution.rs

@@ -10,11 +10,11 @@ use deno_core::serde_json::Map;
 use deno_core::serde_json::Value;
 use deno_core::url::Url;
 use deno_core::ModuleSpecifier;
-use path_clean::PathClean;
 use regex::Regex;
 
 use crate::errors;
 use crate::package_json::PackageJson;
+use crate::path::PathClean;
 use crate::RequireNpmResolver;
 
 pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"];

test_util/src/lib.rs

@@ -1861,6 +1861,7 @@ pub struct CheckOutputIntegrationTest<'a> {
   pub http_server: bool,
   pub envs: Vec<(String, String)>,
   pub env_clear: bool,
+  pub temp_cwd: bool,
 }
 
 impl<'a> CheckOutputIntegrationTest<'a> {
@@ -1874,6 +1875,11 @@ impl<'a> CheckOutputIntegrationTest<'a> {
       );
       std::borrow::Cow::Borrowed(&self.args_vec)
     };
+    let testdata_dir = testdata_path();
+    let args = args
+      .iter()
+      .map(|arg| arg.replace("$TESTDATA", &testdata_dir.to_string_lossy()))
+      .collect::<Vec<_>>();
     let deno_exe = deno_exe_path();
     println!("deno_exe path {}", deno_exe.display());
 
@@ -1884,17 +1890,21 @@ impl<'a> CheckOutputIntegrationTest<'a> {
     };
 
     let (mut reader, writer) = pipe().unwrap();
-    let testdata_dir = testdata_path();
     let deno_dir = new_deno_dir(); // keep this alive for the test
     let mut command = deno_cmd_with_deno_dir(&deno_dir);
-    println!("deno_exe args {}", self.args);
+    let cwd = if self.temp_cwd {
-    println!("deno_exe testdata path {:?}", &testdata_dir);
+      deno_dir.path()
+    } else {
+      testdata_dir.as_path()
+    };
+    println!("deno_exe args {}", args.join(" "));
+    println!("deno_exe cwd {:?}", &testdata_dir);
     command.args(args.iter());
     if self.env_clear {
       command.env_clear();
     }
     command.envs(self.envs.clone());
-    command.current_dir(&testdata_dir);
+    command.current_dir(&cwd);
     command.stdin(Stdio::piped());
     let writer_clone = writer.try_clone().unwrap();
     command.stderr(writer_clone);
@@ -1949,7 +1959,7 @@ impl<'a> CheckOutputIntegrationTest<'a> {
     // deno test's output capturing flushes with a zero-width space in order to
     // synchronize the output pipes. Occassionally this zero width space
     // might end up in the output so strip it from the output comparison here.
-    if args.first() == Some(&"test") {
+    if args.first().map(|s| s.as_str()) == Some("test") {
       actual = actual.replace('\u{200B}', "");
     }