foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-03 04:48:52 -05:00
Code Issues Packages 1 Wiki Activity

refactor: use sys_traits (#27480)

Browse source
This commit is contained in:
David Sherret 2024-12-30 12:38:20 -05:00 committed by GitHub
parent fd8400eaec
commit c391ad315e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
66 changed files with 1378 additions and 1598 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
Cargo.lock generated
View file

@ -867,6 +867,7 @@ dependencies = [
"regex",
"reqwest",
"serde",
"sys_traits",
"test_server",
"tokio",
"url",
@ -1274,7 +1275,7 @@ dependencies = [
"deno_npm",
"deno_npm_cache",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_resolver",
"deno_runtime",
"deno_semver",
@ -1340,6 +1341,7 @@ dependencies = [
"spki",
"sqlformat",
"strsim",
"sys_traits",
"tar",
"tempfile",
"test_server",
@ -1452,9 +1454,9 @@ dependencies = [
[[package]]
name = "deno_cache_dir"
version = "0.15.0"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54df1c5177ace01d92b872584ab9af8290681bb150fd9b423c37a494ad5ddbdc"
checksum = "e73ed17f285731a23df9779ca1e0e721de866db6776ed919ebd9235e0a107c4c"
dependencies = [
"async-trait",
"base32",
@ -1465,7 +1467,7 @@ dependencies = [
"data-url",
"deno_error",
"deno_media_type",
"deno_path_util",
"deno_path_util 0.3.0",
"http 1.1.0",
"indexmap 2.3.0",
"log",
@ -1474,6 +1476,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 1.0.64",
"url",
]
@ -1491,13 +1494,13 @@ dependencies = [
[[package]]
name = "deno_config"
version = "0.41.0"
version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8afa3beb6b9e0604cfe0380d30f88c5b758d44e228d5a5fc42ae637ccfb7d089"
checksum = "b45aaf31e58ca915d5c0746bf8e2d07b94635154ad9e5afe5ff265cae6187b19"
dependencies = [
"anyhow",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"glob",
"ignore",
@ -1509,6 +1512,7 @@ dependencies = [
"phf",
"serde",
"serde_json",
"sys_traits",
"thiserror 1.0.64",
"url",
]
@ -1623,7 +1627,7 @@ dependencies = [
"comrak",
"deno_ast",
"deno_graph",
"deno_path_util",
"deno_path_util 0.2.2",
"handlebars",
"html-escape",
"import_map",
@ -1673,7 +1677,7 @@ dependencies = [
"bytes",
"data-url",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"dyn-clone",
@ -1730,15 +1734,17 @@ dependencies = [
"boxed_error",
"deno_core",
"deno_io",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"filetime",
"getrandom",
"junction",
"libc",
"nix",
"rand",
"rayon",
"serde",
"sys_traits",
"thiserror 2.0.3",
"winapi",
"windows-sys 0.59.0",
@ -1746,15 +1752,16 @@ dependencies = [
[[package]]
name = "deno_graph"
version = "0.86.5"
version = "0.86.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f669d96d63841d9ba10f86b161d898678ce05bc1e3c9ee1c1f7449a68eed2b64"
checksum = "83af194ca492ea7b624d21055f933676d3f3d27586de93be31c8f1babcc73510"
dependencies = [
"anyhow",
"async-trait",
"capacity_builder 0.5.0",
"data-url",
"deno_ast",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"encoding_rs",
@ -1769,6 +1776,7 @@ dependencies = [
"serde",
"serde_json",
"sha2",
"sys_traits",
"thiserror 2.0.3",
"twox-hash",
"url",
@ -1847,7 +1855,7 @@ dependencies = [
"chrono",
"deno_core",
"deno_fetch",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_tls",
"denokv_proto",
@ -1976,7 +1984,7 @@ dependencies = [
"deno_media_type",
"deno_net",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_whoami",
"der",
@ -2078,6 +2086,7 @@ dependencies = [
"deno_core",
"deno_error",
"deno_npm",
"deno_path_util 0.3.0",
"deno_semver",
"deno_unsync",
"faster-hex",
@ -2090,6 +2099,7 @@ dependencies = [
"rand",
"ring",
"serde_json",
"sys_traits",
"tar",
"tempfile",
"thiserror 2.0.3",
@ -2114,17 +2124,18 @@ dependencies = [
[[package]]
name = "deno_package_json"
version = "0.3.0"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81d72db99fdebfc371d7be16972c18a47daa7a29cb5fbb3900ab2114b1f42d96"
checksum = "e1d3c0f699ba2040669204ce24ab73720499fc290af843e4ce0fc8a9b3d67735"
dependencies = [
"boxed_error",
"deno_error",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"indexmap 2.3.0",
"serde",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"url",
]
@ -2141,13 +2152,26 @@ dependencies = [
"url",
]
[[package]]
name = "deno_path_util"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "420e8211aaba7fde83ccaa9a5dad855c3b940ed988d70c95159acd600a70dc87"
dependencies = [
"deno_error",
"percent-encoding",
"sys_traits",
"thiserror 2.0.3",
"url",
]
[[package]]
name = "deno_permissions"
version = "0.43.0"
dependencies = [
"capacity_builder 0.5.0",
"deno_core",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_terminal 0.2.0",
"fqdn",
"libc",
@ -2171,9 +2195,10 @@ dependencies = [
"deno_config",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_semver",
"node_resolver",
"sys_traits",
"test_server",
"thiserror 2.0.3",
"url",
@ -2201,7 +2226,7 @@ dependencies = [
"deno_napi",
"deno_net",
"deno_node",
"deno_path_util",
"deno_path_util 0.3.0",
"deno_permissions",
"deno_telemetry",
"deno_terminal 0.2.0",
@ -5049,13 +5074,14 @@ dependencies = [
"boxed_error",
"deno_media_type",
"deno_package_json",
"deno_path_util",
"deno_path_util 0.3.0",
"futures",
"lazy-regex",
"once_cell",
"path-clean",
"regex",
"serde_json",
"sys_traits",
"thiserror 2.0.3",
"tokio",
"url",
@ -7652,6 +7678,17 @@ dependencies = [
"syn 2.0.87",
]
[[package]]
name = "sys_traits"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5a12729b699487bb50163466e87be7197871d83d04cc6815d430cf7c893bbd7"
dependencies = [
"getrandom",
"libc",
"windows-sys 0.59.0",
]
[[package]]
name = "tagptr"
version = "0.2.0"

9
Cargo.toml
View file

@ -51,11 +51,11 @@ deno_ast = { version = "=0.44.0", features = ["transpiling"] }
deno_core = { version = "0.327.0" }
deno_bench_util = { version = "0.178.0", path = "./bench_util" }
deno_config = { version = "=0.41.0", features = ["workspace", "sync"] }
deno_config = { version = "=0.42.0", features = ["workspace", "sync"] }
deno_lockfile = "=0.24.0"
deno_media_type = { version = "0.2.0", features = ["module_specifier"] }
deno_npm = "=0.27.0"
deno_path_util = "=0.2.2"
deno_path_util = "=0.3.0"
deno_permissions = { version = "0.43.0", path = "./runtime/permissions" }
deno_runtime = { version = "0.192.0", path = "./runtime" }
deno_semver = "=0.7.1"
@ -118,9 +118,9 @@ console_static_text = "=0.8.1"
dashmap = "5.5.3"
data-encoding = "2.3.3"
data-url = "=0.3.1"
deno_cache_dir = "=0.15.0"
deno_cache_dir = "=0.16.0"
deno_error = "=0.5.2"
deno_package_json = { version = "0.3.0", default-features = false }
deno_package_json = { version = "0.4.0", default-features = false }
deno_unsync = "0.4.2"
dlopen2 = "0.6.1"
ecb = "=0.1.2"
@ -193,6 +193,7 @@ slab = "0.4"
smallvec = "1.8"
socket2 = { version = "0.5.3", features = ["all"] }
spki = "0.7.2"
sys_traits = "=0.1.1"
tar = "=0.4.40"
tempfile = "3.4.0"
termcolor = "1.1.3"

3
cli/Cargo.toml
View file

@ -74,7 +74,7 @@ deno_config.workspace = true
deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
deno_doc = { version = "=0.161.3", features = ["rust", "comrak"] }
deno_error.workspace = true
deno_graph = { version = "=0.86.5" }
deno_graph = { version = "=0.86.6" }
deno_lint = { version = "=0.68.2", features = ["docs"] }
deno_lockfile.workspace = true
deno_npm.workspace = true
@ -158,6 +158,7 @@ shell-escape = "=0.1.5"
spki = { version = "0.7", features = ["pem"] }
sqlformat = "=0.3.2"
strsim = "0.11.1"
sys_traits = { workspace = true, features = ["libc", "real", "winapi"] }
tar.workspace = true
tempfile.workspace = true
text-size = "=1.1.0"

56
cli/args/deno_json.rs
View file

@ -8,62 +8,6 @@ use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
#[cfg(test)] // happens to only be used by the tests at the moment
pub struct DenoConfigFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
#[cfg(test)]
impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> {
fn read_to_string_lossy(
&self,
path: &std::path::Path,
) -> Result<std::borrow::Cow<'static, str>, std::io::Error> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn stat_sync(
&self,
path: &std::path::Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.stat_sync(path)
.map(|stat| deno_config::fs::FsMetadata {
is_file: stat.is_file,
is_directory: stat.is_directory,
is_symlink: stat.is_symlink,
})
.map_err(|err| err.into_io_error())
}
fn read_dir(
&self,
path: &std::path::Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
self
.0
.read_dir_sync(path)
.map_err(|err| err.into_io_error())
.map(|entries| {
entries
.into_iter()
.map(|e| deno_config::fs::FsDirEntry {
path: path.join(e.name),
metadata: deno_config::fs::FsMetadata {
is_file: e.is_file,
is_directory: e.is_directory,
is_symlink: e.is_symlink,
},
})
.collect()
})
}
}
pub fn import_map_deps(
import_map: &serde_json::Value,
) -> HashSet<JsrDepPackageReq> {

6
cli/args/lockfile.rs
View file

@ -12,12 +12,13 @@ use deno_core::parking_lot::MutexGuard;
use deno_core::serde_json;
use deno_lockfile::WorkspaceMemberConfig;
use deno_package_json::PackageJsonDepValue;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::PackageJson;
use deno_semver::jsr::JsrDepPackageReq;
use crate::args::deno_json::import_map_deps;
use crate::cache;
use crate::util::fs::atomic_write_file_with_retries;
use crate::Flags;
use crate::args::DenoSubcommand;
@ -91,8 +92,9 @@ impl CliLockfile {
// do an atomic write to reduce the chance of multiple deno
// processes corrupting the file
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&lockfile.filename,
bytes,
&bytes,
cache::CACHE_PERM,
)
.context("Failed writing lockfile.")?;

8
cli/args/mod.rs
View file

@ -30,6 +30,7 @@ use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
use deno_npm::NpmSystemInfo;
use deno_path_util::normalize_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::StackString;
use deno_telemetry::OtelConfig;
@ -83,9 +84,9 @@ use std::num::NonZeroUsize;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::EnvHomeDir;
use thiserror::Error;
use crate::cache;
use crate::cache::DenoDirProvider;
use crate::file_fetcher::CliFileFetcher;
use crate::util::fs::canonicalize_path_maybe_not_exists;
@ -572,7 +573,7 @@ fn discover_npmrc(
// TODO(bartlomieju): update to read both files - one in the project root and one and
// home dir and then merge them.
// 3. Try `.npmrc` in the user's home directory
if let Some(home_dir) = cache::home_dir() {
if let Some(home_dir) = sys_traits::impls::RealSys.env_home_dir() {
match try_to_read_npmrc(&home_dir) {
Ok(Some((source, path))) => {
return try_to_parse_npmrc(source, &path).map(|r| (r, Some(path)));
@ -845,7 +846,6 @@ impl CliOptions {
log::debug!("package.json auto-discovery is disabled");
}
WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs
deno_json_cache: None,
pkg_json_cache: Some(&node_resolver::PackageJsonThreadLocalCache),
workspace_cache: None,
@ -867,6 +867,7 @@ impl CliOptions {
ConfigFlag::Discover => {
if let Some(start_paths) = flags.config_path_args(&initial_cwd) {
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
WorkspaceDiscoverStart::Paths(&start_paths),
&resolve_workspace_discover_options(),
)?
@ -877,6 +878,7 @@ impl CliOptions {
ConfigFlag::Path(path) => {
let config_path = normalize_path(initial_cwd.join(path));
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
WorkspaceDiscoverStart::ConfigFile(&config_path),
&resolve_workspace_discover_options(),
)?

153
cli/cache/deno_dir.rs vendored
View file

@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_cache_dir::DenoDirResolutionError;
use once_cell::sync::OnceCell;
use super::DiskCache;
@ -11,7 +12,7 @@ use std::path::PathBuf;
/// where functionality wants to continue if the DENO_DIR can't be created.
pub struct DenoDirProvider {
maybe_custom_root: Option<PathBuf>,
deno_dir: OnceCell<std::io::Result<DenoDir>>,
deno_dir: OnceCell<Result<DenoDir, DenoDirResolutionError>>,
}
impl DenoDirProvider {
@ -22,12 +23,21 @@ impl DenoDirProvider {
}
}
pub fn get_or_create(&self) -> Result<&DenoDir, std::io::Error> {
pub fn get_or_create(&self) -> Result<&DenoDir, DenoDirResolutionError> {
self
.deno_dir
.get_or_init(|| DenoDir::new(self.maybe_custom_root.clone()))
.as_ref()
.map_err(|err| std::io::Error::new(err.kind(), err.to_string()))
.map_err(|err| match err {
DenoDirResolutionError::NoCacheOrHomeDir => {
DenoDirResolutionError::NoCacheOrHomeDir
}
DenoDirResolutionError::FailedCwd { source } => {
DenoDirResolutionError::FailedCwd {
source: std::io::Error::new(source.kind(), source.to_string()),
}
}
})
}
}
@ -42,27 +52,13 @@ pub struct DenoDir {
}
impl DenoDir {
pub fn new(maybe_custom_root: Option<PathBuf>) -> std::io::Result<Self> {
let maybe_custom_root =
maybe_custom_root.or_else(|| env::var("DENO_DIR").map(String::into).ok());
let root: PathBuf = if let Some(root) = maybe_custom_root {
root
} else if let Some(cache_dir) = dirs::cache_dir() {
// We use the OS cache dir because all files deno writes are cache files
// Once that changes we need to start using different roots if DENO_DIR
// is not set, and keep a single one if it is.
cache_dir.join("deno")
} else if let Some(home_dir) = dirs::home_dir() {
// fallback path
home_dir.join(".deno")
} else {
panic!("Could not set the Deno root directory")
};
let root = if root.is_absolute() {
root
} else {
std::env::current_dir()?.join(root)
};
pub fn new(
maybe_custom_root: Option<PathBuf>,
) -> Result<Self, deno_cache_dir::DenoDirResolutionError> {
let root = deno_cache_dir::resolve_deno_dir(
&sys_traits::impls::RealSys,
maybe_custom_root,
)?;
assert!(root.is_absolute());
let gen_path = root.join("gen");
@ -166,112 +162,3 @@ impl DenoDir {
self.root.join("dl")
}
}
/// To avoid the poorly managed dirs crate
#[cfg(not(windows))]
pub mod dirs {
use std::path::PathBuf;
pub fn cache_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
home_dir().map(|h| h.join("Library/Caches"))
} else {
std::env::var_os("XDG_CACHE_HOME")
.map(PathBuf::from)
.or_else(|| home_dir().map(|h| h.join(".cache")))
}
}
pub fn home_dir() -> Option<PathBuf> {
std::env::var_os("HOME")
.and_then(|h| if h.is_empty() { None } else { Some(h) })
.or_else(|| {
// TODO(bartlomieju):
#[allow(clippy::undocumented_unsafe_blocks)]
unsafe {
fallback()
}
})
.map(PathBuf::from)
}
// This piece of code is taken from the deprecated home_dir() function in Rust's standard library: https://github.com/rust-lang/rust/blob/master/src/libstd/sys/unix/os.rs#L579
// The same code is used by the dirs crate
unsafe fn fallback() -> Option<std::ffi::OsString> {
let amt = match libc::sysconf(libc::_SC_GETPW_R_SIZE_MAX) {
n if n < 0 => 512_usize,
n => n as usize,
};
let mut buf = Vec::with_capacity(amt);
let mut passwd: libc::passwd = std::mem::zeroed();
let mut result = std::ptr::null_mut();
match libc::getpwuid_r(
libc::getuid(),
&mut passwd,
buf.as_mut_ptr(),
buf.capacity(),
&mut result,
) {
0 if !result.is_null() => {
let ptr = passwd.pw_dir as *const _;
let bytes = std::ffi::CStr::from_ptr(ptr).to_bytes().to_vec();
Some(std::os::unix::ffi::OsStringExt::from_vec(bytes))
}
_ => None,
}
}
}
/// To avoid the poorly managed dirs crate
// Copied from
// https://github.com/dirs-dev/dirs-sys-rs/blob/ec7cee0b3e8685573d847f0a0f60aae3d9e07fa2/src/lib.rs#L140-L164
// MIT license. Copyright (c) 2018-2019 dirs-rs contributors
#[cfg(windows)]
pub mod dirs {
use std::ffi::OsString;
use std::os::windows::ffi::OsStringExt;
use std::path::PathBuf;
use winapi::shared::winerror;
use winapi::um::combaseapi;
use winapi::um::knownfolders;
use winapi::um::shlobj;
use winapi::um::shtypes;
use winapi::um::winbase;
use winapi::um::winnt;
fn known_folder(folder_id: shtypes::REFKNOWNFOLDERID) -> Option<PathBuf> {
// SAFETY: winapi calls
unsafe {
let mut path_ptr: winnt::PWSTR = std::ptr::null_mut();
let result = shlobj::SHGetKnownFolderPath(
folder_id,
0,
std::ptr::null_mut(),
&mut path_ptr,
);
if result == winerror::S_OK {
let len = winbase::lstrlenW(path_ptr) as usize;
let path = std::slice::from_raw_parts(path_ptr, len);
let ostr: OsString = OsStringExt::from_wide(path);
combaseapi::CoTaskMemFree(path_ptr as *mut winapi::ctypes::c_void);
Some(PathBuf::from(ostr))
} else {
None
}
}
}
pub fn cache_dir() -> Option<PathBuf> {
known_folder(&knownfolders::FOLDERID_LocalAppData)
}
pub fn home_dir() -> Option<PathBuf> {
if let Some(userprofile) = std::env::var_os("USERPROFILE") {
if !userprofile.is_empty() {
return Some(PathBuf::from(userprofile));
}
}
known_folder(&knownfolders::FOLDERID_Profile)
}
}

10
cli/cache/disk_cache.rs vendored
View file

@ -1,11 +1,12 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use super::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use deno_cache_dir::url_to_filename;
use deno_core::url::Host;
use deno_core::url::Url;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use std::ffi::OsStr;
use std::fs;
use std::path::Component;
@ -120,7 +121,12 @@ impl DiskCache {
pub fn set(&self, filename: &Path, data: &[u8]) -> std::io::Result<()> {
let path = self.location.join(filename);
atomic_write_file_with_retries(&path, data, CACHE_PERM)
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&path,
data,
CACHE_PERM,
)
}
}

137
cli/cache/mod.rs vendored
View file

@ -5,9 +5,6 @@ use crate::file_fetcher::CliFetchNoFollowErrorKind;
use crate::file_fetcher::CliFileFetcher;
use crate::file_fetcher::FetchNoFollowOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use deno_ast::MediaType;
use deno_cache_dir::file_fetcher::CacheSetting;
@ -21,15 +18,12 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_permissions::PermissionsContainer;
use node_resolver::InNpmPackageChecker;
use std::borrow::Cow;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::SystemTime;
mod cache_db;
mod caches;
@ -50,7 +44,6 @@ pub use caches::Caches;
pub use check::TypeCheckCache;
pub use code_cache::CodeCache;
pub use common::FastInsecureHasher;
pub use deno_dir::dirs::home_dir;
pub use deno_dir::DenoDir;
pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
@ -63,121 +56,12 @@ pub use parsed_source::LazyGraphSourceParser;
pub use parsed_source::ParsedSourceCache;
/// Permissions used to save a file in the disk caches.
pub const CACHE_PERM: u32 = 0o644;
pub use deno_cache_dir::CACHE_PERM;
#[derive(Debug, Clone)]
pub struct RealDenoCacheEnv;
impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
std::fs::read(path).map(Cow::Owned)
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries(path, bytes, CACHE_PERM)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
crate::util::fs::canonicalize_path(path)
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(path)
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
match std::fs::metadata(path) {
Ok(metadata) => Ok(Some(
metadata.modified().unwrap_or_else(|_| SystemTime::now()),
)),
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(err) => Err(err),
}
}
fn is_file(&self, path: &Path) -> bool {
path.is_file()
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
#[derive(Debug, Clone)]
pub struct DenoCacheEnvFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> {
fn read_file_bytes(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, [u8]>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.0,
write_mode: CACHE_PERM,
},
path,
bytes,
)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, true, None)
.map_err(|e| e.into_io_error())
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
self
.0
.stat_sync(path)
.map(|stat| {
stat
.mtime
.map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts))
})
.map_err(|e| e.into_io_error())
}
fn is_file(&self, path: &Path) -> bool {
self.0.is_file_sync(path)
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<FsSysTraitsAdapter>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<FsSysTraitsAdapter>;
pub type LocalLspHttpCache =
deno_cache_dir::LocalLspHttpCache<RealDenoCacheEnv>;
deno_cache_dir::LocalLspHttpCache<FsSysTraitsAdapter>;
pub use deno_cache_dir::HttpCache;
pub struct FetchCacherOptions {
@ -192,11 +76,11 @@ pub struct FetchCacherOptions {
pub struct FetchCacher {
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
sys: FsSysTraitsAdapter,
is_deno_publish: bool,
cache_info_enabled: bool,
}
@ -204,18 +88,18 @@ pub struct FetchCacher {
impl FetchCacher {
pub fn new(
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
sys: FsSysTraitsAdapter,
options: FetchCacherOptions,
) -> Self {
Self {
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
module_info_cache,
sys,
file_header_overrides: options.file_header_overrides,
permissions: options.permissions,
is_deno_publish: options.is_deno_publish,
@ -277,9 +161,8 @@ impl Loader for FetchCacher {
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
// against the canonicalized specifier.
let specifier = crate::node::resolve_specifier_into_node_modules(
specifier,
self.fs.as_ref(),
let specifier = node_resolver::resolve_specifier_into_node_modules(
&self.sys, specifier,
);
if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(

40
cli/factory.rs
View file

@ -11,7 +11,6 @@ use crate::args::StorageKeyResolver;
use crate::args::TsConfigType;
use crate::cache::Caches;
use crate::cache::CodeCache;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDir;
use crate::cache::DenoDirProvider;
use crate::cache::EmitCache;
@ -43,7 +42,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
@ -76,9 +74,10 @@ use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -318,8 +317,8 @@ impl CliFactory {
pub fn global_http_cache(&self) -> Result<&Arc<GlobalHttpCache>, AnyError> {
self.services.global_http_cache.get_or_try_init(|| {
Ok(Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter(self.fs().clone()),
self.deno_dir()?.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
)))
})
}
@ -396,7 +395,7 @@ impl CliFactory {
let global_path = self.deno_dir()?.npm_folder_path();
let cli_options = self.cli_options()?;
Ok(Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&FsSysTraitsAdapter(fs.clone()),
global_path,
cli_options.npmrc().get_all_known_registries_urls(),
)))
@ -416,7 +415,7 @@ impl CliFactory {
create_cli_npm_resolver(if cli_options.use_byonm() {
CliNpmResolverCreateOptions::Byonm(
CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: FsSysTraitsAdapter(fs.clone()),
pkg_json_resolver: self.pkg_json_resolver().clone(),
root_node_modules_dir: Some(
match cli_options.node_modules_dir_path() {
@ -434,6 +433,13 @@ impl CliFactory {
} else {
CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
http_client_provider: self.http_client_provider().clone(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
sys: FsSysTraitsAdapter(self.fs().clone()),
snapshot: match cli_options.resolve_npm_resolution_snapshot()? {
Some(snapshot) => {
CliNpmResolverManagedSnapshotOption::Specified(Some(
@ -452,19 +458,12 @@ impl CliFactory {
},
},
maybe_lockfile: cli_options.maybe_lockfile().cloned(),
fs: fs.clone(),
http_client_provider: self.http_client_provider().clone(),
npm_cache_dir: self.npm_cache_dir()?.clone(),
cache_setting: cli_options.cache_setting(),
text_only_progress_bar: self.text_only_progress_bar().clone(),
maybe_node_modules_path: cli_options
.node_modules_dir_path()
.cloned(),
npm_install_deps_provider: Arc::new(
NpmInstallDepsProvider::from_workspace(
cli_options.workspace(),
),
),
npm_system_info: cli_options.npm_system_info(),
npmrc: cli_options.npmrc().clone(),
lifecycle_scripts: cli_options.lifecycle_scripts_config(),
@ -487,7 +486,7 @@ impl CliFactory {
.get_or_try_init(|| {
Ok(self.cli_options()?.unstable_sloppy_imports().then(|| {
Arc::new(CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(
self.fs().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
}))
})
@ -655,14 +654,15 @@ impl CliFactory {
.get_or_try_init_async(
async {
Ok(Arc::new(NodeResolver::new(
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
RealIsBuiltInNodeModuleChecker,
self
.npm_resolver()
.await?
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
}
.boxed_local(),
@ -690,7 +690,6 @@ impl CliFactory {
Ok(Arc::new(NodeCodeTranslator::new(
cjs_esm_analyzer,
DenoFsNodeResolverEnv::new(self.fs().clone()),
self.in_npm_pkg_checker()?.clone(),
node_resolver,
self
@ -699,6 +698,7 @@ impl CliFactory {
.clone()
.into_npm_pkg_folder_resolver(),
self.pkg_json_resolver().clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
})
.await
@ -714,7 +714,7 @@ impl CliFactory {
let npm_resolver = self.npm_resolver().await?;
Ok(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs().clone()),
sys: FsSysTraitsAdapter(self.fs().clone()),
in_npm_pkg_checker: self.in_npm_pkg_checker()?.clone(),
node_resolver: self.node_resolver().await?.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -725,7 +725,7 @@ impl CliFactory {
pub fn pkg_json_resolver(&self) -> &Arc<PackageJsonResolver> {
self.services.pkg_json_resolver.get_or_init(|| {
Arc::new(PackageJsonResolver::new(DenoFsNodeResolverEnv::new(
Arc::new(PackageJsonResolver::new(FsSysTraitsAdapter(
self.fs().clone(),
)))
})
@ -765,7 +765,6 @@ impl CliFactory {
self.cjs_tracker()?.clone(),
cli_options.clone(),
self.file_fetcher()?.clone(),
self.fs().clone(),
self.global_http_cache()?.clone(),
self.in_npm_pkg_checker()?.clone(),
cli_options.maybe_lockfile().cloned(),
@ -775,6 +774,7 @@ impl CliFactory {
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
self.root_permissions_container()?.clone(),
FsSysTraitsAdapter(self.fs().clone()),
)))
})
.await
@ -960,7 +960,6 @@ impl CliFactory {
None
},
self.emitter()?.clone(),
fs.clone(),
in_npm_pkg_checker.clone(),
self.main_module_graph_container().await?.clone(),
self.module_load_preparer().await?.clone(),
@ -975,6 +974,7 @@ impl CliFactory {
),
self.parsed_source_cache().clone(),
self.resolver().await?.clone(),
FsSysTraitsAdapter(self.fs().clone()),
)),
node_resolver.clone(),
npm_resolver.clone(),

55
cli/file_fetcher.rs
View file

@ -1,11 +1,8 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::cache::HttpCache;
use crate::cache::RealDenoCacheEnv;
use crate::colors;
use crate::http_util::get_response_body_with_progress;
use crate::http_util::HttpClientProvider;
use crate::util::progress_bar::ProgressBar;
use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use boxed_error::Boxed;
use deno_ast::MediaType;
@ -27,7 +24,7 @@ use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_error::JsError;
use deno_graph::source::LoaderChecksum;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_permissions::CheckSpecifierKind;
use deno_runtime::deno_permissions::PermissionCheckError;
use deno_runtime::deno_permissions::PermissionsContainer;
@ -35,12 +32,14 @@ use deno_runtime::deno_web::BlobStore;
use http::header;
use http::HeaderMap;
use http::StatusCode;
use std::borrow::Cow;
use std::collections::HashMap;
use std::env;
use std::sync::Arc;
use thiserror::Error;
use crate::cache::HttpCache;
use crate::colors;
use crate::http_util::get_response_body_with_progress;
use crate::http_util::HttpClientProvider;
use crate::util::progress_bar::ProgressBar;
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct TextDecodedFile {
pub media_type: MediaType,
@ -268,7 +267,7 @@ pub struct FetchNoFollowOptions<'a> {
type DenoCacheDirFileFetcher = deno_cache_dir::file_fetcher::FileFetcher<
BlobStoreAdapter,
RealDenoCacheEnv,
FsSysTraitsAdapter,
HttpClientAdapter,
>;
@ -290,9 +289,11 @@ impl CliFileFetcher {
download_log_level: log::Level,
) -> Self {
let memory_files = Arc::new(MemoryFiles::default());
let sys = FsSysTraitsAdapter::new_real();
let auth_tokens = AuthTokens::new_from_sys(&sys);
let file_fetcher = DenoCacheDirFileFetcher::new(
BlobStoreAdapter(blob_store),
RealDenoCacheEnv,
sys,
http_cache,
HttpClientAdapter {
http_client_provider: http_client_provider.clone(),
@ -303,7 +304,7 @@ impl CliFileFetcher {
FileFetcherOptions {
allow_remote,
cache_setting,
auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()),
auth_tokens,
},
);
Self {
@ -497,7 +498,6 @@ fn validate_scheme(specifier: &Url) -> Result<(), UnsupportedSchemeError> {
#[cfg(test)]
mod tests {
use crate::cache::GlobalHttpCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use super::*;
@ -538,7 +538,10 @@ mod tests {
let temp_dir = maybe_temp_dir.unwrap_or_default();
let location = temp_dir.path().join("remote").to_path_buf();
let blob_store: Arc<BlobStore> = Default::default();
let cache = Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv));
let cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
));
let file_fetcher = CliFileFetcher::new(
cache.clone(),
Arc::new(HttpClientProvider::new(None, None)),
@ -752,8 +755,8 @@ mod tests {
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -781,8 +784,8 @@ mod tests {
resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap();
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let file_modified_01 = {
let file_fetcher = CliFileFetcher::new(
@ -808,8 +811,8 @@ mod tests {
let file_modified_02 = {
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -938,8 +941,8 @@ mod tests {
let redirected_specifier =
resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap();
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let metadata_file_modified_01 = {
@ -1073,8 +1076,8 @@ mod tests {
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
crate::cache::RealDenoCacheEnv,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
@ -1110,7 +1113,10 @@ mod tests {
let temp_dir = TempDir::new();
let location = temp_dir.path().join("remote").to_path_buf();
let file_fetcher_01 = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)),
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
None,
@ -1119,7 +1125,10 @@ mod tests {
log::Level::Info,
);
let file_fetcher_02 = CliFileFetcher::new(
Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)),
Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location,
)),
Arc::new(HttpClientProvider::new(None, None)),
Default::default(),
None,

164
cli/graph_util.rs
View file

@ -1,5 +1,43 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::source::ResolveError;
use deno_graph::FillFromLockfileOptions;
use deno_graph::GraphKind;
use deno_graph::JsrLoadError;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ModuleLoadError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_graph::WorkspaceFastCheckOption;
use deno_path_util::url_to_file_path;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::SmallStackString;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use crate::args::config_to_deno_graph_workspace_member;
use crate::args::jsr_url;
use crate::args::CliLockfile;
@ -23,43 +61,6 @@ use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::file_watcher::WatcherCommunicator;
use crate::util::fs::canonicalize_path;
use deno_config::deno_json::JsxImportSourceConfig;
use deno_config::workspace::JsrPackageConfig;
use deno_core::anyhow::bail;
use deno_graph::source::LoaderChecksum;
use deno_graph::source::ResolutionKind;
use deno_graph::FillFromLockfileOptions;
use deno_graph::JsrLoadError;
use deno_graph::ModuleLoadError;
use deno_graph::WorkspaceFastCheckOption;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier;
use deno_graph::source::Loader;
use deno_graph::source::ResolveError;
use deno_graph::GraphKind;
use deno_graph::ModuleError;
use deno_graph::ModuleGraph;
use deno_graph::ModuleGraphError;
use deno_graph::ResolutionError;
use deno_graph::SpecifierError;
use deno_path_util::url_to_file_path;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_semver::jsr::JsrDepPackageReq;
use deno_semver::package::PackageNv;
use deno_semver::SmallStackString;
use import_map::ImportMapError;
use node_resolver::InNpmPackageChecker;
use std::collections::HashSet;
use std::error::Error;
use std::ops::Deref;
use std::path::PathBuf;
use std::sync::Arc;
#[derive(Clone)]
pub struct GraphValidOptions {
@ -80,7 +81,7 @@ pub struct GraphValidOptions {
/// for the CLI.
pub fn graph_valid(
graph: &ModuleGraph,
fs: &Arc<dyn FileSystem>,
sys: &FsSysTraitsAdapter,
roots: &[ModuleSpecifier],
options: GraphValidOptions,
) -> Result<(), AnyError> {
@ -90,7 +91,7 @@ pub fn graph_valid(
let mut errors = graph_walk_errors(
graph,
fs,
sys,
roots,
GraphWalkErrorsOptions {
check_js: options.check_js,
@ -140,7 +141,7 @@ pub struct GraphWalkErrorsOptions {
/// and enhances them with CLI information.
pub fn graph_walk_errors<'a>(
graph: &'a ModuleGraph,
fs: &'a Arc<dyn FileSystem>,
sys: &'a FsSysTraitsAdapter,
roots: &'a [ModuleSpecifier],
options: GraphWalkErrorsOptions,
) -> impl Iterator<Item = AnyError> + 'a {
@ -175,7 +176,7 @@ pub fn graph_walk_errors<'a>(
}
ModuleGraphError::ModuleError(error) => {
enhanced_integrity_error_message(error)
.or_else(|| enhanced_sloppy_imports_error_message(fs, error))
.or_else(|| enhanced_sloppy_imports_error_message(sys, error))
.unwrap_or_else(|| format_deno_graph_error(error))
}
};
@ -433,7 +434,6 @@ pub struct ModuleGraphBuilder {
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -443,6 +443,7 @@ pub struct ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: FsSysTraitsAdapter,
}
impl ModuleGraphBuilder {
@ -452,7 +453,6 @@ impl ModuleGraphBuilder {
cjs_tracker: Arc<CjsTracker>,
cli_options: Arc<CliOptions>,
file_fetcher: Arc<CliFileFetcher>,
fs: Arc<dyn FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
lockfile: Option<Arc<CliLockfile>>,
@ -462,13 +462,13 @@ impl ModuleGraphBuilder {
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
root_permissions_container: PermissionsContainer,
sys: FsSysTraitsAdapter,
) -> Self {
Self {
caches,
cjs_tracker,
cli_options,
file_fetcher,
fs,
global_http_cache,
in_npm_pkg_checker,
lockfile,
@ -478,6 +478,7 @@ impl ModuleGraphBuilder {
parsed_source_cache,
resolver,
root_permissions_container,
sys,
}
}
@ -593,7 +594,7 @@ impl ModuleGraphBuilder {
is_dynamic: options.is_dynamic,
passthrough_jsr_specifiers: false,
executor: Default::default(),
file_system: &DenoGraphFsAdapter(self.fs.as_ref()),
file_system: &self.sys,
jsr_url_provider: &CliJsrUrlProvider,
npm_resolver: Some(&graph_npm_resolver),
module_analyzer: &analyzer,
@ -747,10 +748,10 @@ impl ModuleGraphBuilder {
) -> cache::FetchCacher {
cache::FetchCacher::new(
self.file_fetcher.clone(),
self.fs.clone(),
self.global_http_cache.clone(),
self.in_npm_pkg_checker.clone(),
self.module_info_cache.clone(),
self.sys.clone(),
cache::FetchCacherOptions {
file_header_overrides: self.cli_options.resolve_file_header_overrides(),
permissions,
@ -779,7 +780,7 @@ impl ModuleGraphBuilder {
) -> Result<(), AnyError> {
graph_valid(
graph,
&self.fs,
&self.sys,
roots,
GraphValidOptions {
kind: if self.cli_options.type_check_mode().is_true() {
@ -835,13 +836,13 @@ pub fn enhanced_resolution_error_message(error: &ResolutionError) -> String {
}
fn enhanced_sloppy_imports_error_message(
fs: &Arc<dyn FileSystem>,
sys: &FsSysTraitsAdapter,
error: &ModuleError,
) -> Option<String> {
match error {
ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error
| ModuleError::Missing(specifier, _) => {
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(fs.clone()))
let additional_message = CliSloppyImportsResolver::new(SloppyImportsCachedFs::new(sys.clone()))
.resolve(specifier, SloppyImportsResolutionKind::Execution)?
.as_suggestion_message();
Some(format!(
@ -1082,71 +1083,6 @@ impl deno_graph::source::Reporter for FileWatcherReporter {
}
}
pub struct DenoGraphFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_graph::source::FileSystem for DenoGraphFsAdapter<'a> {
fn read_dir(
&self,
dir_url: &deno_graph::ModuleSpecifier,
) -> Vec<deno_graph::source::DirEntry> {
use deno_core::anyhow;
use deno_graph::source::DirEntry;
use deno_graph::source::DirEntryKind;
let dir_path = match dir_url.to_file_path() {
Ok(path) => path,
// ignore, treat as non-analyzable
Err(()) => return vec![],
};
let entries = match self.0.read_dir_sync(&dir_path) {
Ok(dir) => dir,
Err(err)
if matches!(
err.kind(),
std::io::ErrorKind::PermissionDenied | std::io::ErrorKind::NotFound
) =>
{
return vec![];
}
Err(err) => {
return vec![DirEntry {
kind: DirEntryKind::Error(
anyhow::Error::from(err)
.context("Failed to read directory.".to_string()),
),
url: dir_url.clone(),
}];
}
};
let mut dir_entries = Vec::with_capacity(entries.len());
for entry in entries {
let entry_path = dir_path.join(&entry.name);
dir_entries.push(if entry.is_directory {
DirEntry {
kind: DirEntryKind::Dir,
url: ModuleSpecifier::from_directory_path(&entry_path).unwrap(),
}
} else if entry.is_file {
DirEntry {
kind: DirEntryKind::File,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else if entry.is_symlink {
DirEntry {
kind: DirEntryKind::Symlink,
url: ModuleSpecifier::from_file_path(&entry_path).unwrap(),
}
} else {
continue;
});
}
dir_entries
}
}
pub fn format_range_with_colors(referrer: &deno_graph::Range) -> String {
format!(
"{}:{}:{}",

3
cli/lsp/cache.rs
View file

@ -11,6 +11,7 @@ use crate::lsp::logging::lsp_warn;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
@ -94,8 +95,8 @@ impl LspCache {
let deno_dir = DenoDir::new(global_cache_path)
.expect("should be infallible with absolute custom root");
let global = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
deno_dir.remote_folder_path(),
crate::cache::RealDenoCacheEnv,
));
Self {
deno_dir,

93
cli/lsp/config.rs
View file

@ -9,8 +9,6 @@ use deno_config::deno_json::LintConfig;
use deno_config::deno_json::NodeModulesDirMode;
use deno_config::deno_json::TestConfig;
use deno_config::deno_json::TsConfig;
use deno_config::fs::DenoConfigFs;
use deno_config::fs::RealDenoConfigFs;
use deno_config::glob::FilePatterns;
use deno_config::glob::PathOrPatternSet;
use deno_config::workspace::CreateResolverOptions;
@ -38,10 +36,10 @@ use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_package_json::PackageJsonCache;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::PackageJson;
use indexmap::IndexSet;
use lsp_types::ClientCapabilities;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::collections::HashMap;
@ -1220,7 +1218,6 @@ impl ConfigData {
settings: &Settings,
file_fetcher: &Arc<CliFileFetcher>,
// sync requirement is because the lsp requires sync
cached_deno_config_fs: &(dyn DenoConfigFs + Sync),
deno_json_cache: &(dyn DenoJsonCache + Sync),
pkg_json_cache: &(dyn PackageJsonCache + Sync),
workspace_cache: &(dyn WorkspaceCache + Sync),
@ -1230,6 +1227,7 @@ impl ConfigData {
Ok(scope_dir_path) => {
let paths = [scope_dir_path];
WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
match specified_config {
Some(config_path) => {
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
@ -1241,7 +1239,6 @@ impl ConfigData {
}
},
&WorkspaceDiscoverOptions {
fs: cached_deno_config_fs,
additional_config_file_names: &[],
deno_json_cache: Some(deno_json_cache),
pkg_json_cache: Some(pkg_json_cache),
@ -1618,9 +1615,9 @@ impl ConfigData {
|| unstable.contains("sloppy-imports");
let sloppy_imports_resolver = unstable_sloppy_imports.then(|| {
Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new_without_stat_cache(Arc::new(
deno_runtime::deno_fs::RealFs,
)),
SloppyImportsCachedFs::new_without_stat_cache(
FsSysTraitsAdapter::new_real(),
),
))
});
let resolver = Arc::new(resolver);
@ -1840,7 +1837,6 @@ impl ConfigTree {
// since we're resolving a workspace multiple times in different
// folders, we want to cache all the lookups and config files across
// ConfigData::load calls
let cached_fs = CachedDenoConfigFs::default();
let deno_json_cache = DenoJsonMemCache::default();
let pkg_json_cache = PackageJsonMemCache::default();
let workspace_cache = WorkspaceMemCache::default();
@ -1865,7 +1861,6 @@ impl ConfigTree {
folder_uri,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1896,7 +1891,6 @@ impl ConfigTree {
&scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1913,7 +1907,6 @@ impl ConfigTree {
member_scope,
settings,
file_fetcher,
&cached_fs,
&deno_json_cache,
&pkg_json_cache,
&workspace_cache,
@ -1930,7 +1923,7 @@ impl ConfigTree {
pub async fn inject_config_file(&mut self, config_file: ConfigFile) {
let scope = config_file.specifier.join(".").unwrap();
let json_text = serde_json::to_string(&config_file.json).unwrap();
let test_fs = deno_runtime::deno_fs::InMemoryFs::default();
let test_fs = Arc::new(deno_runtime::deno_fs::InMemoryFs::default());
let config_path = url_to_file_path(&config_file.specifier).unwrap();
test_fs.setup_text_files(vec![(
config_path.to_string_lossy().to_string(),
@ -1938,11 +1931,11 @@ impl ConfigTree {
)]);
let workspace_dir = Arc::new(
WorkspaceDirectory::discover(
&FsSysTraitsAdapter(test_fs.clone()),
deno_config::workspace::WorkspaceDiscoverStart::ConfigFile(
&config_path,
),
&deno_config::workspace::WorkspaceDiscoverOptions {
fs: &crate::args::deno_json::DenoConfigFsAdapter(&test_fs),
..Default::default()
},
)
@ -2076,78 +2069,6 @@ impl deno_config::workspace::WorkspaceCache for WorkspaceMemCache {
}
}
#[derive(Default)]
struct CachedFsItems<T: Clone> {
items: HashMap<PathBuf, Result<T, std::io::Error>>,
}
impl<T: Clone> CachedFsItems<T> {
pub fn get(
&mut self,
path: &Path,
action: impl FnOnce(&Path) -> Result<T, std::io::Error>,
) -> Result<T, std::io::Error> {
let value = if let Some(value) = self.items.get(path) {
value
} else {
let value = action(path);
// just in case this gets really large for some reason
if self.items.len() == 16_384 {
return value;
}
self.items.insert(path.to_owned(), value);
self.items.get(path).unwrap()
};
value
.as_ref()
.map(|v| (*v).clone())
.map_err(|e| std::io::Error::new(e.kind(), e.to_string()))
}
}
#[derive(Default)]
struct InnerData {
stat_calls: CachedFsItems<deno_config::fs::FsMetadata>,
read_to_string_calls: CachedFsItems<Cow<'static, str>>,
}
#[derive(Default)]
struct CachedDenoConfigFs(Mutex<InnerData>);
impl DenoConfigFs for CachedDenoConfigFs {
fn stat_sync(
&self,
path: &Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.lock()
.stat_calls
.get(path, |path| RealDenoConfigFs.stat_sync(path))
}
fn read_to_string_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, std::io::Error> {
self
.0
.lock()
.read_to_string_calls
.get(path, |path| RealDenoConfigFs.read_to_string_lossy(path))
}
fn read_dir(
&self,
path: &Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
// no need to cache these because the workspace cache will ensure
// we only do read_dir calls once (read_dirs are only used for
// npm workspace resolution)
RealDenoConfigFs.read_dir(path)
}
}
#[cfg(test)]
mod tests {
use deno_config::deno_json::ConfigParseOptions;

4
cli/lsp/diagnostics.rs
View file

@ -48,7 +48,7 @@ use deno_graph::SpecifierError;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_resolver::sloppy_imports::SloppyImportsResolution;
use deno_resolver::sloppy_imports::SloppyImportsResolutionKind;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node;
use deno_runtime::tokio_util::create_basic_runtime;
use deno_semver::jsr::JsrPackageReqReference;
@ -1281,7 +1281,7 @@ impl DenoDiagnostic {
Self::NotInstalledNpm(pkg_req, specifier) => (lsp::DiagnosticSeverity::ERROR, format!("npm package \"{pkg_req}\" is not installed or doesn't exist."), Some(json!({ "specifier": specifier }))),
Self::NoLocal(specifier) => {
let maybe_sloppy_resolution = CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(Arc::new(deno_fs::RealFs))
SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real())
).resolve(specifier, SloppyImportsResolutionKind::Execution);
let data = maybe_sloppy_resolution.as_ref().map(|res| {
json!({

5
cli/lsp/language_server.rs
View file

@ -17,6 +17,7 @@ use deno_core::ModuleSpecifier;
use deno_graph::GraphKind;
use deno_graph::Resolution;
use deno_path_util::url_to_file_path;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::RootCertStoreProvider;
use deno_semver::jsr::JsrPackageReqReference;
@ -279,7 +280,7 @@ impl LanguageServer {
.await?;
graph_util::graph_valid(
&graph,
factory.fs(),
&FsSysTraitsAdapter(factory.fs().clone()),
&roots,
graph_util::GraphValidOptions {
kind: GraphKind::All,
@ -3612,11 +3613,11 @@ impl Inner {
let workspace = match config_data {
Some(d) => d.member_dir.clone(),
None => Arc::new(WorkspaceDirectory::discover(
&FsSysTraitsAdapter::new_real(),
deno_config::workspace::WorkspaceDiscoverStart::Paths(&[
initial_cwd.clone()
]),
&WorkspaceDiscoverOptions {
fs: Default::default(), // use real fs,
deno_json_cache: None,
pkg_json_cache: None,
workspace_cache: None,

3
cli/lsp/registries.rs
View file

@ -32,6 +32,7 @@ use deno_core::url::Position;
use deno_core::url::Url;
use deno_core::ModuleSpecifier;
use deno_graph::Dependency;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::error;
use once_cell::sync::Lazy;
use std::borrow::Cow;
@ -430,8 +431,8 @@ impl ModuleRegistry {
) -> Self {
// the http cache should always be the global one for registry completions
let http_cache = Arc::new(GlobalHttpCache::new(
FsSysTraitsAdapter::new_real(),
location.clone(),
crate::cache::RealDenoCacheEnv,
));
let file_fetcher = CliFileFetcher::new(
http_cache.clone(),

37
cli/lsp/resolver.rs
View file

@ -19,9 +19,10 @@ use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_resolver::DenoResolverOptions;
use deno_resolver::NodeAndNpmReqResolver;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::jsr::JsrPackageReqReference;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageNv;
@ -42,7 +43,6 @@ use super::jsr::JsrCacheResolver;
use crate::args::create_default_npmrc;
use crate::args::CliLockfile;
use crate::args::NpmInstallDepsProvider;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::factory::Deferred;
use crate::graph_util::to_node_resolution_kind;
use crate::graph_util::to_node_resolution_mode;
@ -61,7 +61,6 @@ use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::npm::ManagedCliNpmResolver;
use crate::resolver::CliDenoResolver;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::CliResolverOptions;
@ -599,21 +598,19 @@ struct ResolverFactoryServices {
struct ResolverFactory<'a> {
config_data: Option<&'a Arc<ConfigData>>,
fs: Arc<dyn deno_fs::FileSystem>,
pkg_json_resolver: Arc<PackageJsonResolver>,
sys: FsSysTraitsAdapter,
services: ResolverFactoryServices,
}
impl<'a> ResolverFactory<'a> {
pub fn new(config_data: Option<&'a Arc<ConfigData>>) -> Self {
let fs = Arc::new(deno_fs::RealFs);
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let sys = FsSysTraitsAdapter::new_real();
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone()));
Self {
config_data,
fs,
pkg_json_resolver,
sys,
services: Default::default(),
}
}
@ -624,9 +621,10 @@ impl<'a> ResolverFactory<'a> {
cache: &LspCache,
) {
let enable_byonm = self.config_data.map(|d| d.byonm).unwrap_or(false);
let sys = FsSysTraitsAdapter::new_real();
let options = if enable_byonm {
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)),
sys,
pkg_json_resolver: self.pkg_json_resolver.clone(),
root_node_modules_dir: self.config_data.and_then(|config_data| {
config_data.node_modules_dir.clone().or_else(|| {
@ -642,12 +640,14 @@ impl<'a> ResolverFactory<'a> {
.and_then(|d| d.npmrc.clone())
.unwrap_or_else(create_default_npmrc);
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(self.fs.as_ref()),
&sys,
cache.deno_dir().npm_folder_path(),
npmrc.get_all_known_registries_urls(),
));
CliNpmResolverCreateOptions::Managed(CliManagedNpmResolverCreateOptions {
http_client_provider: http_client_provider.clone(),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
snapshot: match self.config_data.and_then(|d| d.lockfile.as_ref()) {
Some(lockfile) => {
CliNpmResolverManagedSnapshotOption::ResolveFromLockfile(
@ -656,10 +656,7 @@ impl<'a> ResolverFactory<'a> {
}
None => CliNpmResolverManagedSnapshotOption::Specified(None),
},
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
fs: Arc::new(deno_fs::RealFs),
sys: FsSysTraitsAdapter::new_real(),
npm_cache_dir,
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
@ -667,11 +664,12 @@ impl<'a> ResolverFactory<'a> {
// the user is typing.
cache_setting: CacheSetting::Only,
text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly),
// Don't provide the lockfile. We don't want these resolvers
// updating it. Only the cache request should update the lockfile.
maybe_lockfile: None,
maybe_node_modules_path: self
.config_data
.and_then(|d| d.node_modules_dir.clone()),
// only used for top level install, so we can ignore this
npm_install_deps_provider: Arc::new(NpmInstallDepsProvider::empty()),
npmrc,
npm_system_info: NpmSystemInfo::default(),
lifecycle_scripts: Default::default(),
@ -779,10 +777,11 @@ impl<'a> ResolverFactory<'a> {
.get_or_init(|| {
let npm_resolver = self.services.npm_resolver.as_ref()?;
Some(Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(self.fs.clone()),
self.in_npm_pkg_checker().clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
self.pkg_json_resolver.clone(),
self.sys.clone(),
)))
})
.as_ref()
@ -797,10 +796,10 @@ impl<'a> ResolverFactory<'a> {
let npm_resolver = self.npm_resolver()?;
Some(Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(self.fs.clone()),
in_npm_pkg_checker: self.in_npm_pkg_checker().clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
sys: self.sys.clone(),
})))
})
.as_ref()

79
cli/module_loader.rs
View file

@ -11,36 +11,6 @@ use std::sync::atomic::AtomicU16;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
use deno_ast::MediaType;
use deno_ast::ModuleKind;
use deno_core::anyhow::anyhow;
@ -69,7 +39,7 @@ use deno_graph::ModuleGraph;
use deno_graph::Resolution;
use deno_graph::WasmModule;
use deno_runtime::code_cache;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
@ -79,6 +49,37 @@ use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use sys_traits::FsRead;
use crate::args::jsr_url;
use crate::args::CliLockfile;
use crate::args::CliOptions;
use crate::args::DenoSubcommand;
use crate::args::TsTypeLib;
use crate::cache::CodeCache;
use crate::cache::FastInsecureHasher;
use crate::cache::ParsedSourceCache;
use crate::emit::Emitter;
use crate::graph_container::MainModuleGraphContainer;
use crate::graph_container::ModuleGraphContainer;
use crate::graph_container::ModuleGraphUpdatePermit;
use crate::graph_util::CreateGraphOptions;
use crate::graph_util::ModuleGraphBuilder;
use crate::node::CliNodeCodeTranslator;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::CliResolver;
use crate::resolver::ModuleCodeStringSource;
use crate::resolver::NotSupportedKindInNpmError;
use crate::resolver::NpmModuleLoader;
use crate::tools::check;
use crate::tools::check::TypeChecker;
use crate::util::progress_bar::ProgressBar;
use crate::util::text_encoding::code_without_source_map;
use crate::util::text_encoding::source_map_from_code;
use crate::worker::CreateModuleLoaderResult;
use crate::worker::ModuleLoaderFactory;
pub struct ModuleLoadPreparer {
options: Arc<CliOptions>,
@ -215,7 +216,6 @@ struct SharedCliModuleLoaderState {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
@ -226,6 +226,7 @@ struct SharedCliModuleLoaderState {
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: FsSysTraitsAdapter,
in_flight_loads_tracker: InFlightModuleLoadsTracker,
}
@ -275,7 +276,6 @@ impl CliModuleLoaderFactory {
cjs_tracker: Arc<CjsTracker>,
code_cache: Option<Arc<CodeCache>>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
main_module_graph_container: Arc<MainModuleGraphContainer>,
module_load_preparer: Arc<ModuleLoadPreparer>,
@ -286,6 +286,7 @@ impl CliModuleLoaderFactory {
npm_module_loader: NpmModuleLoader,
parsed_source_cache: Arc<ParsedSourceCache>,
resolver: Arc<CliResolver>,
sys: FsSysTraitsAdapter,
) -> Self {
Self {
shared: Arc::new(SharedCliModuleLoaderState {
@ -301,7 +302,6 @@ impl CliModuleLoaderFactory {
cjs_tracker,
code_cache,
emitter,
fs,
in_npm_pkg_checker,
main_module_graph_container,
module_load_preparer,
@ -312,6 +312,7 @@ impl CliModuleLoaderFactory {
npm_module_loader,
parsed_source_cache,
resolver,
sys,
in_flight_loads_tracker: InFlightModuleLoadsTracker {
loads_number: Arc::new(AtomicU16::new(0)),
cleanup_task_timeout: 10_000,
@ -344,7 +345,7 @@ impl CliModuleLoaderFactory {
let node_require_loader = Rc::new(CliNodeRequireLoader {
cjs_tracker: self.shared.cjs_tracker.clone(),
emitter: self.shared.emitter.clone(),
fs: self.shared.fs.clone(),
sys: self.shared.sys.clone(),
graph_container,
in_npm_pkg_checker: self.shared.in_npm_pkg_checker.clone(),
npm_resolver: self.shared.npm_resolver.clone(),
@ -593,9 +594,9 @@ impl<TGraphContainer: ModuleGraphContainer>
Some(Module::Json(module)) => module.specifier.clone(),
Some(Module::Wasm(module)) => module.specifier.clone(),
Some(Module::External(module)) => {
node::resolve_specifier_into_node_modules(
node_resolver::resolve_specifier_into_node_modules(
&self.shared.sys,
&module.specifier,
self.shared.fs.as_ref(),
)
}
None => specifier.into_owned(),
@ -1091,7 +1092,7 @@ impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit {
struct CliNodeRequireLoader<TGraphContainer: ModuleGraphContainer> {
cjs_tracker: Arc<CjsTracker>,
emitter: Arc<Emitter>,
fs: Arc<dyn FileSystem>,
sys: FsSysTraitsAdapter,
graph_container: TGraphContainer,
in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
npm_resolver: Arc<dyn CliNpmResolver>,
@ -1120,7 +1121,7 @@ impl<TGraphContainer: ModuleGraphContainer> NodeRequireLoader
) -> Result<Cow<'static, str>, AnyError> {
// todo(dsherret): use the preloaded module from the graph if available?
let media_type = MediaType::from_path(path);
let text = self.fs.read_text_file_lossy_sync(path, None)?;
let text = self.sys.fs_read_to_string_lossy(path)?;
if media_type.is_emittable() {
let specifier = deno_path_util::url_from_file_path(path)?;
if self.in_npm_pkg_checker.in_npm_package(&specifier) {

25
cli/node.rs
View file

@ -8,7 +8,8 @@ use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_graph::ParsedSourceStore;
use deno_runtime::deno_fs;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis;
use node_resolver::analyze::CjsAnalysisExports;
use node_resolver::analyze::CjsCodeAnalyzer;
@ -21,23 +22,11 @@ use crate::cache::NodeAnalysisCache;
use crate::cache::ParsedSourceCache;
use crate::resolver::CjsTracker;
pub type CliNodeCodeTranslator =
NodeCodeTranslator<CliCjsCodeAnalyzer, DenoFsNodeResolverEnv>;
/// Resolves a specifier that is pointing into a node_modules folder.
///
/// Note: This should be called whenever getting the specifier from
/// a Module::External(module) reference because that module might
/// not be fully resolved at the time deno_graph is analyzing it
/// because the node_modules folder might not exist at that time.
pub fn resolve_specifier_into_node_modules(
specifier: &ModuleSpecifier,
fs: &dyn deno_fs::FileSystem,
) -> ModuleSpecifier {
node_resolver::resolve_specifier_into_node_modules(specifier, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
}
pub type CliNodeCodeTranslator = NodeCodeTranslator<
CliCjsCodeAnalyzer,
RealIsBuiltInNodeModuleChecker,
FsSysTraitsAdapter,
>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum CliCjsAnalysis {

8
cli/npm/byonm.rs
View file

@ -9,22 +9,20 @@ use deno_core::serde_json;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::ByonmNpmResolverCreateOptions;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use node_resolver::NpmPackageFolderResolver;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::resolver::CliDenoResolverFs;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
pub type CliByonmNpmResolverCreateOptions =
ByonmNpmResolverCreateOptions<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub type CliByonmNpmResolver =
ByonmNpmResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
ByonmNpmResolverCreateOptions<FsSysTraitsAdapter>;
pub type CliByonmNpmResolver = ByonmNpmResolver<FsSysTraitsAdapter>;
// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
#[derive(Debug)]

100
cli/npm/managed/mod.rs
View file

@ -21,9 +21,10 @@ use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_npm_cache::NpmCacheSetting;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::CliNpmReqResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -41,7 +42,6 @@ use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::args::PackageJsonDepValueParseWithLocationError;
use crate::cache::FastInsecureHasher;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::progress_bar::ProgressBar;
use crate::util::sync::AtomicFlag;
@ -50,7 +50,7 @@ use self::resolvers::create_npm_fs_resolver;
use self::resolvers::NpmPackageFsResolver;
use super::CliNpmCache;
use super::CliNpmCacheEnv;
use super::CliNpmCacheHttpClient;
use super::CliNpmRegistryInfoProvider;
use super::CliNpmResolver;
use super::CliNpmTarballCache;
@ -68,9 +68,9 @@ pub enum CliNpmResolverManagedSnapshotOption {
pub struct CliManagedNpmResolverCreateOptions {
pub snapshot: CliNpmResolverManagedSnapshotOption,
pub maybe_lockfile: Option<Arc<CliLockfile>>,
pub fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
pub npm_cache_dir: Arc<NpmCacheDir>,
pub sys: FsSysTraitsAdapter,
pub cache_setting: deno_cache_dir::file_fetcher::CacheSetting,
pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
pub maybe_node_modules_path: Option<PathBuf>,
@ -83,9 +83,12 @@ pub struct CliManagedNpmResolverCreateOptions {
pub async fn create_managed_npm_resolver_for_lsp(
options: CliManagedNpmResolverCreateOptions,
) -> Arc<dyn CliNpmResolver> {
let cache_env = create_cache_env(&options);
let npm_cache = create_cache(cache_env.clone(), &options);
let npm_api = create_api(npm_cache.clone(), cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let npm_api = create_api(npm_cache.clone(), http_client.clone(), &options);
// spawn due to the lsp's `Send` requirement
deno_core::unsync::spawn(async move {
let snapshot = match resolve_snapshot(&npm_api, options.snapshot).await {
@ -96,14 +99,14 @@ pub async fn create_managed_npm_resolver_for_lsp(
}
};
create_inner(
cache_env,
options.fs,
options.maybe_lockfile,
npm_api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
npm_api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -117,19 +120,22 @@ pub async fn create_managed_npm_resolver_for_lsp(
pub async fn create_managed_npm_resolver(
options: CliManagedNpmResolverCreateOptions,
) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
let npm_cache_env = create_cache_env(&options);
let npm_cache = create_cache(npm_cache_env.clone(), &options);
let api = create_api(npm_cache.clone(), npm_cache_env.clone(), &options);
let npm_cache = create_cache(&options);
let http_client = Arc::new(CliNpmCacheHttpClient::new(
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
));
let api = create_api(npm_cache.clone(), http_client.clone(), &options);
let snapshot = resolve_snapshot(&api, options.snapshot).await?;
Ok(create_inner(
npm_cache_env,
options.fs,
options.maybe_lockfile,
api,
http_client,
npm_cache,
options.npmrc,
options.npm_install_deps_provider,
api,
options.sys,
options.text_only_progress_bar,
options.maybe_lockfile,
options.npmrc,
options.maybe_node_modules_path,
options.npm_system_info,
snapshot,
@ -139,14 +145,14 @@ pub async fn create_managed_npm_resolver(
#[allow(clippy::too_many_arguments)]
fn create_inner(
env: Arc<CliNpmCacheEnv>,
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
http_client: Arc<CliNpmCacheHttpClient>,
npm_cache: Arc<CliNpmCache>,
npm_rc: Arc<ResolvedNpmRc>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
sys: FsSysTraitsAdapter,
text_only_progress_bar: crate::util::progress_bar::ProgressBar,
maybe_lockfile: Option<Arc<CliLockfile>>,
npm_rc: Arc<ResolvedNpmRc>,
node_modules_dir_path: Option<PathBuf>,
npm_system_info: NpmSystemInfo,
snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
@ -159,28 +165,29 @@ fn create_inner(
));
let tarball_cache = Arc::new(CliNpmTarballCache::new(
npm_cache.clone(),
env,
http_client,
sys.clone(),
npm_rc.clone(),
));
let fs_resolver = create_npm_fs_resolver(
fs.clone(),
npm_cache.clone(),
&npm_install_deps_provider,
&text_only_progress_bar,
resolution.clone(),
sys.clone(),
tarball_cache.clone(),
node_modules_dir_path,
npm_system_info.clone(),
lifecycle_scripts.clone(),
);
Arc::new(ManagedCliNpmResolver::new(
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
npm_cache,
npm_install_deps_provider,
resolution,
sys,
tarball_cache,
text_only_progress_bar,
npm_system_info,
@ -188,36 +195,25 @@ fn create_inner(
))
}
fn create_cache_env(
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCacheEnv> {
Arc::new(CliNpmCacheEnv::new(
options.fs.clone(),
options.http_client_provider.clone(),
options.text_only_progress_bar.clone(),
))
}
fn create_cache(
env: Arc<CliNpmCacheEnv>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmCache> {
Arc::new(CliNpmCache::new(
options.npm_cache_dir.clone(),
options.sys.clone(),
NpmCacheSetting::from_cache_setting(&options.cache_setting),
env,
options.npmrc.clone(),
))
}
fn create_api(
cache: Arc<CliNpmCache>,
env: Arc<CliNpmCacheEnv>,
http_client: Arc<CliNpmCacheHttpClient>,
options: &CliManagedNpmResolverCreateOptions,
) -> Arc<CliNpmRegistryInfoProvider> {
Arc::new(CliNpmRegistryInfoProvider::new(
cache,
env,
http_client,
options.npmrc.clone(),
))
}
@ -306,12 +302,12 @@ pub enum PackageCaching<'a> {
/// An npm resolver where the resolution is managed by Deno rather than
/// the user bringing their own node_modules (BYONM) on the file system.
pub struct ManagedCliNpmResolver {
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
sys: FsSysTraitsAdapter,
resolution: Arc<NpmResolution>,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
@ -331,20 +327,19 @@ impl std::fmt::Debug for ManagedCliNpmResolver {
impl ManagedCliNpmResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
fs: Arc<dyn FileSystem>,
fs_resolver: Arc<dyn NpmPackageFsResolver>,
maybe_lockfile: Option<Arc<CliLockfile>>,
registry_info_provider: Arc<CliNpmRegistryInfoProvider>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
text_only_progress_bar: ProgressBar,
npm_system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
fs,
fs_resolver,
maybe_lockfile,
registry_info_provider,
@ -352,6 +347,7 @@ impl ManagedCliNpmResolver {
npm_install_deps_provider,
text_only_progress_bar,
resolution,
sys,
tarball_cache,
npm_system_info,
top_level_install_flag: Default::default(),
@ -364,8 +360,7 @@ impl ManagedCliNpmResolver {
pkg_id: &NpmPackageId,
) -> Result<PathBuf, AnyError> {
let path = self.fs_resolver.package_folder(pkg_id)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())?;
let path = canonicalize_path_maybe_not_exists(&self.sys, &path)?;
log::debug!(
"Resolved package folder of {} to {}",
pkg_id.as_serialized(),
@ -667,12 +662,13 @@ impl NpmPackageFolderResolver for ManagedCliNpmResolver {
.fs_resolver
.resolve_package_folder_from_package(name, referrer)?;
let path =
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map_err(|err| PackageFolderResolveIoError {
canonicalize_path_maybe_not_exists(&self.sys, &path).map_err(|err| {
PackageFolderResolveIoError {
package_name: name.to_string(),
referrer: referrer.clone(),
source: err,
})?;
}
})?;
log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
Ok(path)
}
@ -728,13 +724,12 @@ impl CliNpmResolver for ManagedCliNpmResolver {
));
Arc::new(ManagedCliNpmResolver::new(
self.fs.clone(),
create_npm_fs_resolver(
self.fs.clone(),
self.npm_cache.clone(),
&self.npm_install_deps_provider,
&self.text_only_progress_bar,
npm_resolution.clone(),
self.sys.clone(),
self.tarball_cache.clone(),
self.root_node_modules_path().map(ToOwned::to_owned),
self.npm_system_info.clone(),
@ -745,6 +740,7 @@ impl CliNpmResolver for ManagedCliNpmResolver {
self.npm_cache.clone(),
self.npm_install_deps_provider.clone(),
npm_resolution,
self.sys.clone(),
self.tarball_cache.clone(),
self.text_only_progress_bar.clone(),
self.npm_system_info.clone(),

13
cli/npm/managed/resolvers/common.rs
View file

@ -11,7 +11,6 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::sync::Mutex;
use super::super::PackageCaching;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
@ -21,10 +20,12 @@ use deno_core::futures::StreamExt;
use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use sys_traits::FsCanonicalize;
use super::super::PackageCaching;
use crate::npm::CliNpmTarballCache;
/// Part of the resolution that interacts with the file system.
@ -73,15 +74,15 @@ pub trait NpmPackageFsResolver: Send + Sync {
#[derive(Debug)]
pub struct RegistryReadPermissionChecker {
fs: Arc<dyn FileSystem>,
sys: FsSysTraitsAdapter,
cache: Mutex<HashMap<PathBuf, PathBuf>>,
registry_path: PathBuf,
}
impl RegistryReadPermissionChecker {
pub fn new(fs: Arc<dyn FileSystem>, registry_path: PathBuf) -> Self {
pub fn new(fs: FsSysTraitsAdapter, registry_path: PathBuf) -> Self {
Self {
fs,
sys: fs,
registry_path,
cache: Default::default(),
}
@ -108,7 +109,7 @@ impl RegistryReadPermissionChecker {
|path: &Path| -> Result<Option<PathBuf>, AnyError> {
match cache.get(path) {
Some(canon) => Ok(Some(canon.clone())),
None => match self.fs.realpath_sync(path) {
None => match self.sys.fs_canonicalize(path) {
Ok(canon) => {
cache.insert(path.to_path_buf(), canon.clone());
Ok(Some(canon))

6
cli/npm/managed/resolvers/global.rs
View file

@ -18,7 +18,7 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageNotFoundError;
@ -47,15 +47,15 @@ pub struct GlobalNpmPackageResolver {
impl GlobalNpmPackageResolver {
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn FileSystem>,
tarball_cache: Arc<CliNpmTarballCache>,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
system_info: NpmSystemInfo,
lifecycle_scripts: LifecycleScriptsConfig,
) -> Self {
Self {
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys,
cache.root_dir_path().to_path_buf(),
),
cache,

38
cli/npm/managed/resolvers/local.rs
View file

@ -15,11 +15,6 @@ use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
use crate::args::LifecycleScriptsConfig;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
@ -34,8 +29,10 @@ use deno_npm::NpmPackageCacheFolderId;
use deno_npm::NpmPackageId;
use deno_npm::NpmResolutionPackage;
use deno_npm::NpmSystemInfo;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_path_util::fs::canonicalize_path_maybe_not_exists;
use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_semver::package::PackageNv;
use deno_semver::StackString;
@ -45,11 +42,15 @@ use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::ReferrerNotFoundError;
use serde::Deserialize;
use serde::Serialize;
use sys_traits::FsMetadata;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::colors;
use crate::npm::managed::PackageCaching;
use crate::npm::CliNpmCache;
use crate::npm::CliNpmTarballCache;
use crate::util::fs::clone_dir_recursive;
use crate::util::fs::symlink_dir;
use crate::util::fs::LaxSingleProcessFsFlag;
@ -66,10 +67,10 @@ use super::common::RegistryReadPermissionChecker;
#[derive(Debug)]
pub struct LocalNpmPackageResolver {
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
root_node_modules_path: PathBuf,
root_node_modules_url: Url,
@ -82,10 +83,10 @@ impl LocalNpmPackageResolver {
#[allow(clippy::too_many_arguments)]
pub fn new(
cache: Arc<CliNpmCache>,
fs: Arc<dyn deno_fs::FileSystem>,
npm_install_deps_provider: Arc<NpmInstallDepsProvider>,
progress_bar: ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
node_modules_folder: PathBuf,
system_info: NpmSystemInfo,
@ -93,15 +94,15 @@ impl LocalNpmPackageResolver {
) -> Self {
Self {
cache,
fs: fs.clone(),
npm_install_deps_provider,
progress_bar,
resolution,
tarball_cache,
registry_read_permission_checker: RegistryReadPermissionChecker::new(
fs,
sys.clone(),
node_modules_folder.clone(),
),
sys,
root_node_modules_url: Url::from_directory_path(&node_modules_folder)
.unwrap(),
root_node_modules_path: node_modules_folder,
@ -140,8 +141,7 @@ impl LocalNpmPackageResolver {
};
// Canonicalize the path so it's not pointing to the symlinked directory
// in `node_modules` directory of the referrer.
canonicalize_path_maybe_not_exists_with_fs(&path, self.fs.as_ref())
.map(Some)
canonicalize_path_maybe_not_exists(&self.sys, &path).map(Some)
}
fn resolve_package_folder_from_specifier(
@ -210,7 +210,7 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver {
};
let sub_dir = join_package_name(&node_modules_folder, name);
if self.fs.is_dir_sync(&sub_dir) {
if self.sys.fs_is_dir_no_err(&sub_dir) {
return Ok(sub_dir);
}
@ -925,7 +925,13 @@ impl SetupCache {
}
bincode::serialize(&self.current).ok().and_then(|data| {
atomic_write_file_with_retries(&self.file_path, data, CACHE_PERM).ok()
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
&self.file_path,
&data,
CACHE_PERM,
)
.ok()
});
true
}

8
cli/npm/managed/resolvers/mod.rs
View file

@ -8,7 +8,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use deno_npm::NpmSystemInfo;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
@ -25,11 +25,11 @@ use super::resolution::NpmResolution;
#[allow(clippy::too_many_arguments)]
pub fn create_npm_fs_resolver(
fs: Arc<dyn FileSystem>,
npm_cache: Arc<CliNpmCache>,
npm_install_deps_provider: &Arc<NpmInstallDepsProvider>,
progress_bar: &ProgressBar,
resolution: Arc<NpmResolution>,
sys: FsSysTraitsAdapter,
tarball_cache: Arc<CliNpmTarballCache>,
maybe_node_modules_path: Option<PathBuf>,
system_info: NpmSystemInfo,
@ -38,10 +38,10 @@ pub fn create_npm_fs_resolver(
match maybe_node_modules_path {
Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
npm_cache,
fs,
npm_install_deps_provider.clone(),
progress_bar.clone(),
resolution,
sys,
tarball_cache,
node_modules_folder,
system_info,
@ -49,9 +49,9 @@ pub fn create_npm_fs_resolver(
)),
None => Arc::new(GlobalNpmPackageResolver::new(
npm_cache,
fs,
tarball_cache,
resolution,
sys,
system_info,
lifecycle_scripts,
)),

53
cli/npm/mod.rs
View file

@ -17,7 +17,7 @@ use deno_resolver::npm::ByonmInNpmPackageChecker;
use deno_resolver::npm::ByonmNpmResolver;
use deno_resolver::npm::CliNpmReqResolver;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodePermissions;
use deno_runtime::ops::process::NpmProcessStateProvider;
use deno_semver::package::PackageNv;
@ -30,9 +30,6 @@ use node_resolver::NpmPackageFolderResolver;
use crate::file_fetcher::CliFileFetcher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::hard_link_dir_recursive;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::progress_bar::ProgressBar;
pub use self::byonm::CliByonmNpmResolver;
@ -43,26 +40,26 @@ pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;
pub use self::managed::PackageCaching;
pub type CliNpmTarballCache = deno_npm_cache::TarballCache<CliNpmCacheEnv>;
pub type CliNpmCache = deno_npm_cache::NpmCache<CliNpmCacheEnv>;
pub type CliNpmRegistryInfoProvider =
deno_npm_cache::RegistryInfoProvider<CliNpmCacheEnv>;
pub type CliNpmTarballCache =
deno_npm_cache::TarballCache<CliNpmCacheHttpClient, FsSysTraitsAdapter>;
pub type CliNpmCache = deno_npm_cache::NpmCache<FsSysTraitsAdapter>;
pub type CliNpmRegistryInfoProvider = deno_npm_cache::RegistryInfoProvider<
CliNpmCacheHttpClient,
FsSysTraitsAdapter,
>;
#[derive(Debug)]
pub struct CliNpmCacheEnv {
fs: Arc<dyn FileSystem>,
pub struct CliNpmCacheHttpClient {
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
}
impl CliNpmCacheEnv {
impl CliNpmCacheHttpClient {
pub fn new(
fs: Arc<dyn FileSystem>,
http_client_provider: Arc<HttpClientProvider>,
progress_bar: ProgressBar,
) -> Self {
Self {
fs,
http_client_provider,
progress_bar,
}
@ -70,35 +67,7 @@ impl CliNpmCacheEnv {
}
#[async_trait::async_trait(?Send)]
impl deno_npm_cache::NpmCacheEnv for CliNpmCacheEnv {
fn exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
// todo(dsherret): use self.fs here instead
hard_link_dir_recursive(from, to)
}
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.fs.as_ref(),
write_mode: crate::cache::CACHE_PERM,
},
file_path,
data,
)
}
impl deno_npm_cache::NpmCacheHttpClient for CliNpmCacheHttpClient {
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,

96
cli/resolver.rs
View file

@ -1,5 +1,10 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use async_trait::async_trait;
use dashmap::DashMap;
use dashmap::DashSet;
@ -20,16 +25,14 @@ use deno_npm::resolution::NpmResolutionError;
use deno_resolver::sloppy_imports::SloppyImportsResolver;
use deno_runtime::colors;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::is_builtin_node_module;
use deno_runtime::deno_node::DenoFsNodeResolverEnv;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_semver::package::PackageReq;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use sys_traits::FsMetadata;
use sys_traits::FsMetadataValue;
use thiserror::Error;
use crate::args::NpmCachingStrategy;
@ -40,18 +43,19 @@ use crate::npm::InnerCliNpmResolverRef;
use crate::util::sync::AtomicFlag;
use crate::util::text_encoding::from_utf8_lossy_cow;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<DenoFsNodeResolverEnv>;
pub type IsCjsResolver =
deno_resolver::cjs::IsCjsResolver<DenoFsNodeResolverEnv>;
pub type CjsTracker = deno_resolver::cjs::CjsTracker<FsSysTraitsAdapter>;
pub type IsCjsResolver = deno_resolver::cjs::IsCjsResolver<FsSysTraitsAdapter>;
pub type CliSloppyImportsResolver =
SloppyImportsResolver<SloppyImportsCachedFs>;
pub type CliDenoResolver = deno_resolver::DenoResolver<
CliDenoResolverFs,
DenoFsNodeResolverEnv,
RealIsBuiltInNodeModuleChecker,
SloppyImportsCachedFs,
FsSysTraitsAdapter,
>;
pub type CliNpmReqResolver = deno_resolver::npm::NpmReqResolver<
RealIsBuiltInNodeModuleChecker,
FsSysTraitsAdapter,
>;
pub type CliNpmReqResolver =
deno_resolver::npm::NpmReqResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
pub struct ModuleCodeStringSource {
pub code: ModuleSourceCode,
@ -59,53 +63,6 @@ pub struct ModuleCodeStringSource {
pub media_type: MediaType,
}
#[derive(Debug, Clone)]
pub struct CliDenoResolverFs(pub Arc<dyn FileSystem>);
impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs {
fn read_to_string_lossy(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, str>> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|e| e.into_io_error())
}
fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn exists_sync(&self, path: &Path) -> bool {
self.0.exists_sync(path)
}
fn is_dir_sync(&self, path: &Path) -> bool {
self.0.is_dir_sync(path)
}
fn read_dir_sync(
&self,
dir_path: &Path,
) -> std::io::Result<Vec<deno_resolver::fs::DirEntry>> {
self
.0
.read_dir_sync(dir_path)
.map(|entries| {
entries
.into_iter()
.map(|e| deno_resolver::fs::DirEntry {
name: e.name,
is_file: e.is_file,
is_directory: e.is_directory,
})
.collect::<Vec<_>>()
})
.map_err(|err| err.into_io_error())
}
}
#[derive(Debug, Error)]
#[error("{media_type} files are not supported in npm packages: {specifier}")]
pub struct NotSupportedKindInNpmError {
@ -440,7 +397,7 @@ impl<'a> deno_graph::source::NpmResolver for WorkerCliNpmGraphResolver<'a> {
#[derive(Debug)]
pub struct SloppyImportsCachedFs {
fs: Arc<dyn deno_fs::FileSystem>,
sys: FsSysTraitsAdapter,
cache: Option<
DashMap<
PathBuf,
@ -450,15 +407,18 @@ pub struct SloppyImportsCachedFs {
}
impl SloppyImportsCachedFs {
pub fn new(fs: Arc<dyn FileSystem>) -> Self {
pub fn new(sys: FsSysTraitsAdapter) -> Self {
Self {
fs,
sys,
cache: Some(Default::default()),
}
}
pub fn new_without_stat_cache(fs: Arc<dyn FileSystem>) -> Self {
Self { fs, cache: None }
pub fn new_without_stat_cache(fs: FsSysTraitsAdapter) -> Self {
Self {
sys: fs,
cache: None,
}
}
}
@ -475,10 +435,10 @@ impl deno_resolver::sloppy_imports::SloppyImportResolverFs
}
}
let entry = self.fs.stat_sync(path).ok().and_then(|stat| {
if stat.is_file {
let entry = self.sys.fs_metadata(path).ok().and_then(|stat| {
if stat.file_type().is_file() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::File)
} else if stat.is_directory {
} else if stat.file_type().is_dir() {
Some(deno_resolver::sloppy_imports::SloppyImportsFsEntry::Dir)
} else {
None

1
cli/standalone/binary.rs
View file

@ -37,7 +37,6 @@ use deno_core::futures::AsyncReadExt;
use deno_core::futures::AsyncSeekExt;
use deno_core::serde_json;
use deno_core::url::Url;
use deno_graph::source::RealFileSystem;
use deno_graph::ModuleGraph;
use deno_npm::resolution::SerializedNpmResolutionSnapshot;
use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage;

6
cli/standalone/code_cache.rs
View file

@ -15,11 +15,12 @@ use deno_core::anyhow::bail;
use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
use deno_core::unsync::sync::AtomicFlag;
use deno_path_util::get_atomic_path;
use deno_runtime::code_cache::CodeCache;
use deno_runtime::code_cache::CodeCacheType;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::cache::FastInsecureHasher;
use crate::util::path::get_atomic_file_path;
use crate::worker::CliCodeCache;
enum CodeCacheStrategy {
@ -189,7 +190,8 @@ impl FirstRunCodeCacheStrategy {
cache_data: &HashMap<CodeCacheKey, DenoCompileCodeCacheEntry>,
) {
let count = cache_data.len();
let temp_file = get_atomic_file_path(&self.file_path);
let temp_file =
get_atomic_path(&FsSysTraitsAdapter::new_real(), &self.file_path);
match serialize(&temp_file, self.cache_key, cache_data) {
Ok(()) => {
if let Err(err) = std::fs::rename(&temp_file, &self.file_path) {

1
cli/standalone/file_system.rs
View file

@ -9,6 +9,7 @@ use deno_runtime::deno_fs::AccessCheckCb;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsDirEntry;
use deno_runtime::deno_fs::FsFileType;
use deno_runtime::deno_fs::FsStatSlim;
use deno_runtime::deno_fs::OpenOptions;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_io::fs::File;

47
cli/standalone/mod.rs
View file

@ -36,10 +36,12 @@ use deno_package_json::PackageJsonDepValue;
use deno_resolver::cjs::IsCjsResolutionMode;
use deno_resolver::npm::NpmReqResolverOptions;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::create_host_defined_options;
use deno_runtime::deno_node::NodeRequireLoader;
use deno_runtime::deno_node::NodeResolver;
use deno_runtime::deno_node::PackageJsonResolver;
use deno_runtime::deno_node::RealIsBuiltInNodeModuleChecker;
use deno_runtime::deno_permissions::Permissions;
use deno_runtime::deno_permissions::PermissionsContainer;
use deno_runtime::deno_tls::rustls::RootCertStore;
@ -69,11 +71,9 @@ use crate::args::CaData;
use crate::args::NpmInstallDepsProvider;
use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::DenoDirProvider;
use crate::cache::FastInsecureHasher;
use crate::cache::NodeAnalysisCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use crate::node::CliCjsCodeAnalyzer;
use crate::node::CliNodeCodeTranslator;
@ -87,7 +87,6 @@ use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::CreateInNpmPkgCheckerOptions;
use crate::resolver::CjsTracker;
use crate::resolver::CliDenoResolverFs;
use crate::resolver::CliNpmReqResolver;
use crate::resolver::NpmModuleLoader;
use crate::util::progress_bar::ProgressBar;
@ -656,9 +655,8 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let npm_global_cache_dir = root_path.join(".deno_compile_node_modules");
let cache_setting = CacheSetting::Only;
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
));
let sys = FsSysTraitsAdapter(fs.clone());
let pkg_json_resolver = Arc::new(PackageJsonResolver::new(sys.clone()));
let (in_npm_pkg_checker, npm_resolver) = match metadata.node_modules {
Some(binary::NodeModules::Managed { node_modules_dir }) => {
// create an npmrc that uses the fake npm_registry_url to resolve packages
@ -671,7 +669,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
registry_configs: Default::default(),
});
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -692,17 +690,17 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
snapshot,
)),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
text_only_progress_bar: progress_bar,
cache_setting,
maybe_node_modules_path,
npm_system_info: Default::default(),
npmrc,
lifecycle_scripts: Default::default(),
},
@ -719,7 +717,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_in_npm_pkg_checker(CreateInNpmPkgCheckerOptions::Byonm);
let npm_resolver = create_cli_npm_resolver(
CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions {
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
pkg_json_resolver: pkg_json_resolver.clone(),
root_node_modules_dir,
}),
@ -732,7 +730,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
// so no need to create actual `.npmrc` configuration.
let npmrc = create_default_npmrc();
let npm_cache_dir = Arc::new(NpmCacheDir::new(
&DenoCacheEnvFsAdapter(fs.as_ref()),
&sys,
npm_global_cache_dir,
npmrc.get_all_known_registries_urls(),
));
@ -747,18 +745,18 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
create_cli_npm_resolver(CliNpmResolverCreateOptions::Managed(
CliManagedNpmResolverCreateOptions {
snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
maybe_lockfile: None,
fs: fs.clone(),
http_client_provider: http_client_provider.clone(),
npm_cache_dir,
cache_setting,
text_only_progress_bar: progress_bar,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npm_install_deps_provider: Arc::new(
// this is only used for installing packages, which isn't necessary with deno compile
NpmInstallDepsProvider::empty(),
),
sys: sys.clone(),
cache_setting,
text_only_progress_bar: progress_bar,
npm_cache_dir,
maybe_lockfile: None,
maybe_node_modules_path: None,
npm_system_info: Default::default(),
npmrc: create_default_npmrc(),
lifecycle_scripts: Default::default(),
},
@ -770,10 +768,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some();
let node_resolver = Arc::new(NodeResolver::new(
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker.clone(),
RealIsBuiltInNodeModuleChecker,
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys.clone(),
));
let cjs_tracker = Arc::new(CjsTracker::new(
in_npm_pkg_checker.clone(),
@ -791,7 +790,7 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
let npm_req_resolver =
Arc::new(CliNpmReqResolver::new(NpmReqResolverOptions {
byonm_resolver: (npm_resolver.clone()).into_maybe_byonm(),
fs: CliDenoResolverFs(fs.clone()),
sys: sys.clone(),
in_npm_pkg_checker: in_npm_pkg_checker.clone(),
node_resolver: node_resolver.clone(),
npm_req_resolver: npm_resolver.clone().into_npm_req_resolver(),
@ -804,11 +803,11 @@ pub async fn run(data: StandaloneData) -> Result<i32, AnyError> {
);
let node_code_translator = Arc::new(NodeCodeTranslator::new(
cjs_esm_code_analyzer,
deno_runtime::deno_node::DenoFsNodeResolverEnv::new(fs.clone()),
in_npm_pkg_checker,
node_resolver.clone(),
npm_resolver.clone().into_npm_pkg_folder_resolver(),
pkg_json_resolver.clone(),
sys,
));
let workspace_resolver = {
let import_map = match metadata.workspace_resolver.import_map {

3
cli/tools/coverage/mod.rs
View file

@ -26,6 +26,7 @@ use deno_core::serde_json;
use deno_core::sourcemap::SourceMap;
use deno_core::url::Url;
use deno_core::LocalInspectorSession;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use node_resolver::InNpmPackageChecker;
use regex::Regex;
use std::fs;
@ -428,7 +429,7 @@ fn collect_coverages(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)?;
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns)?;
let coverage_patterns = FilePatterns {
base: initial_cwd.to_path_buf(),

5
cli/tools/doc.rs
View file

@ -28,6 +28,7 @@ use deno_graph::EsParser;
use deno_graph::GraphKind;
use deno_graph::ModuleAnalyzer;
use deno_graph::ModuleSpecifier;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use doc::html::ShortPath;
use doc::DocDiagnostic;
use indexmap::IndexMap;
@ -114,7 +115,7 @@ pub async fn doc(
}
DocSourceFileFlag::Paths(ref source_files) => {
let module_graph_creator = factory.module_graph_creator().await?;
let fs = factory.fs();
let fs = FsSysTraitsAdapter(factory.fs().clone());
let module_specifiers = collect_specifiers(
FilePatterns {
@ -141,7 +142,7 @@ pub async fn doc(
graph_exit_integrity_errors(&graph);
let errors = graph_walk_errors(
&graph,
fs,
&fs,
&module_specifiers,
GraphWalkErrorsOptions {
check_js: false,

3
cli/tools/fmt.rs
View file

@ -34,6 +34,7 @@ use deno_core::futures;
use deno_core::parking_lot::Mutex;
use deno_core::unsync::spawn_blocking;
use deno_core::url::Url;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::debug;
use log::info;
use log::warn;
@ -230,7 +231,7 @@ fn collect_fmt_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)
}
/// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks

6
cli/tools/lint/linter.rs
View file

@ -15,8 +15,9 @@ use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_lint::linter::LintFileOptions;
use deno_lint::linter::Linter as DenoLintLinter;
use deno_lint::linter::LinterOptions;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::specifier_from_file_path;
use super::rules::FileOrPackageLintRule;
@ -176,8 +177,9 @@ impl CliLinter {
if fix_iterations > 0 {
// everything looks good and the file still parses, so write it out
atomic_write_file_with_retries(
&FsSysTraitsAdapter::new_real(),
file_path,
source.text().as_ref(),
source.text().as_bytes(),
crate::cache::CACHE_PERM,
)
.context("Failed writing fix to file.")?;

3
cli/tools/lint/mod.rs
View file

@ -21,6 +21,7 @@ use deno_core::unsync::future::SharedLocal;
use deno_graph::ModuleGraph;
use deno_lint::diagnostic::LintDiagnostic;
use deno_lint::linter::LintConfig as DenoLintConfig;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use log::debug;
use reporters::create_reporter;
use reporters::LintReporter;
@ -452,7 +453,7 @@ fn collect_lint_files(
.ignore_node_modules()
.use_gitignore()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)
}
#[allow(clippy::print_stdout)]

7
cli/tools/registry/paths.rs
View file

@ -11,6 +11,7 @@ use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_core::error::AnyError;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use thiserror::Error;
use crate::args::CliOptions;
@ -323,11 +324,11 @@ fn collect_paths(
file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(|e| {
if !e.metadata.is_file {
if !e.metadata.file_type().is_file() {
if let Ok(specifier) = ModuleSpecifier::from_file_path(e.path) {
diagnostics_collector.push(PublishDiagnostic::UnsupportedFileType {
specifier,
kind: if e.metadata.is_symlink {
kind: if e.metadata.file_type().is_symlink() {
"symlink".to_string()
} else {
"Unknown".to_string()
@ -345,5 +346,5 @@ fn collect_paths(
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.use_gitignore()
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), file_patterns)
}

6
cli/tools/registry/unfurl.rs
View file

@ -663,6 +663,7 @@ mod tests {
use deno_config::workspace::ResolverWorkspaceJsrPackage;
use deno_core::serde_json::json;
use deno_core::url::Url;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_fs::RealFs;
use deno_runtime::deno_node::PackageJson;
use deno_semver::Version;
@ -722,10 +723,9 @@ mod tests {
vec![Arc::new(package_json)],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),
SloppyImportsCachedFs::new(FsSysTraitsAdapter::new_real()),
))),
Arc::new(workspace_resolver),
true,
@ -863,7 +863,7 @@ const warn2 = await import(`${expr}`);
],
deno_config::workspace::PackageJsonDepResolution::Enabled,
);
let fs = Arc::new(RealFs);
let fs = FsSysTraitsAdapter(Arc::new(RealFs));
let unfurler = SpecifierUnfurler::new(
Some(Arc::new(CliSloppyImportsResolver::new(
SloppyImportsCachedFs::new(fs),

12
cli/tsc/mod.rs
View file

@ -4,7 +4,6 @@ use crate::args::TsConfig;
use crate::args::TypeCheckMode;
use crate::cache::FastInsecureHasher;
use crate::cache::ModuleInfoCache;
use crate::node;
use crate::npm::CliNpmResolver;
use crate::resolver::CjsTracker;
use crate::util::checksum;
@ -35,12 +34,13 @@ use deno_graph::Module;
use deno_graph::ModuleGraph;
use deno_graph::ResolutionResolved;
use deno_resolver::npm::ResolvePkgFolderFromDenoReqError;
use deno_runtime::deno_fs;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use deno_runtime::deno_node::NodeResolver;
use deno_semver::npm::NpmPackageReqReference;
use node_resolver::errors::NodeJsErrorCode;
use node_resolver::errors::NodeJsErrorCoded;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::resolve_specifier_into_node_modules;
use node_resolver::NodeResolutionKind;
use node_resolver::ResolutionMode;
use once_cell::sync::Lazy;
@ -660,9 +660,9 @@ fn op_load_inner(
None
} else {
// means it's Deno code importing an npm module
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&FsSysTraitsAdapter::new_real(),
&module.specifier,
&deno_fs::RealFs,
);
Some(Cow::Owned(load_from_node_modules(
&specifier,
@ -924,9 +924,9 @@ fn resolve_graph_specifier_types(
Some(Module::External(module)) => {
// we currently only use "External" for when the module is in an npm package
Ok(state.maybe_npm.as_ref().map(|_| {
let specifier = node::resolve_specifier_into_node_modules(
let specifier = resolve_specifier_into_node_modules(
&FsSysTraitsAdapter::new_real(),
&module.specifier,
&deno_fs::RealFs,
);
into_specifier_and_media_type(Some(specifier))
}))

318
cli/util/fs.rs
View file

@ -1,9 +1,7 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::fs::OpenOptions;
use std::io::Error;
use std::io::ErrorKind;
use std::io::Write;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
@ -19,185 +17,12 @@ use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::unsync::spawn_blocking;
use deno_core::ModuleSpecifier;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_fs::FsSysTraitsAdapter;
use crate::util::path::get_atomic_file_path;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
use crate::util::progress_bar::ProgressMessagePrompt;
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
pub fn atomic_write_file_with_retries<T: AsRef<[u8]>>(
file_path: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
struct RealAtomicWriteFileFs {
mode: u32,
}
impl AtomicWriteFileFs for RealAtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
write_file(path, bytes, self.mode)
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::rename(from, to)
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(dir_path)
}
fn path_exists(&self, path: &Path) -> bool {
path.exists()
}
}
atomic_write_file_with_retries_and_fs(
&RealAtomicWriteFileFs { mode },
file_path,
data.as_ref(),
)
}
pub trait AtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>;
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>;
fn remove_file(&self, path: &Path) -> std::io::Result<()>;
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>;
fn path_exists(&self, path: &Path) -> bool;
}
pub struct AtomicWriteFileFsAdapter<'a> {
pub fs: &'a dyn FileSystem,
pub write_mode: u32,
}
impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
self
.fs
.write_file_sync(
path,
deno_runtime::deno_fs::OpenOptions::write(
true,
false,
false,
Some(self.write_mode),
),
None,
bytes,
)
.map_err(|e| e.into_io_error())
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self.fs.rename_sync(from, to).map_err(|e| e.into_io_error())
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.fs
.remove_sync(path, false)
.map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
self
.fs
.mkdir_sync(dir_path, /* recursive */ true, None)
.map_err(|e| e.into_io_error())
}
fn path_exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
}
pub fn atomic_write_file_with_retries_and_fs<T: AsRef<[u8]>>(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: T,
) -> std::io::Result<()> {
let mut count = 0;
loop {
match atomic_write_file(fs, file_path, data.as_ref()) {
Ok(()) => return Ok(()),
Err(err) => {
if count >= 5 {
// too many retries, return the error
return Err(err);
}
count += 1;
let sleep_ms = std::cmp::min(50, 10 * count);
std::thread::sleep(std::time::Duration::from_millis(sleep_ms));
}
}
}
}
/// Writes the file to the file system at a temporary path, then
/// renames it to the destination in a single sys call in order
/// to never leave the file system in a corrupted state.
///
/// This also handles creating the directory if a NotFound error
/// occurs.
fn atomic_write_file(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fn atomic_write_file_raw(
fs: &impl AtomicWriteFileFs,
temp_file_path: &Path,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()> {
fs.write_file(temp_file_path, data)?;
fs.rename_file(temp_file_path, file_path)
.inspect_err(|_err| {
// clean up the created temp file on error
let _ = fs.remove_file(temp_file_path);
})
}
let temp_file_path = get_atomic_file_path(file_path);
if let Err(write_err) =
atomic_write_file_raw(fs, &temp_file_path, file_path, data)
{
if write_err.kind() == ErrorKind::NotFound {
let parent_dir_path = file_path.parent().unwrap();
match fs.create_dir_all(parent_dir_path) {
Ok(()) => {
return atomic_write_file_raw(fs, &temp_file_path, file_path, data)
.map_err(|err| add_file_context_to_err(file_path, err));
}
Err(create_err) => {
if !fs.path_exists(parent_dir_path) {
return Err(Error::new(
create_err.kind(),
format!(
"{:#} (for '{}')\nCheck the permission of the directory.",
create_err,
parent_dir_path.display()
),
));
}
}
}
}
return Err(add_file_context_to_err(file_path, write_err));
}
Ok(())
}
/// Creates a std::fs::File handling if the parent does not exist.
pub fn create_file(file_path: &Path) -> std::io::Result<std::fs::File> {
match std::fs::File::create(file_path) {
@ -236,45 +61,6 @@ fn add_file_context_to_err(file_path: &Path, err: Error) -> Error {
)
}
pub fn write_file<T: AsRef<[u8]>>(
filename: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
write_file_2(filename, data, true, mode, true, false)
}
pub fn write_file_2<T: AsRef<[u8]>>(
filename: &Path,
data: T,
update_mode: bool,
mode: u32,
is_create: bool,
is_append: bool,
) -> std::io::Result<()> {
let mut file = OpenOptions::new()
.read(false)
.write(true)
.append(is_append)
.truncate(!is_append)
.create(is_create)
.open(filename)?;
if update_mode {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = mode & 0o777;
let permissions = PermissionsExt::from_mode(mode);
file.set_permissions(permissions)?;
}
#[cfg(not(unix))]
let _ = mode;
}
file.write_all(data.as_ref())
}
/// Similar to `std::fs::canonicalize()` but strips UNC prefixes on Windows.
pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
Ok(deno_path_util::strip_unc_prefix(path.canonicalize()?))
@ -289,16 +75,10 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> {
pub fn canonicalize_path_maybe_not_exists(
path: &Path,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path)
}
pub fn canonicalize_path_maybe_not_exists_with_fs(
path: &Path,
fs: &dyn FileSystem,
) -> Result<PathBuf, Error> {
deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| {
fs.realpath_sync(path).map_err(|err| err.into_io_error())
})
deno_path_util::fs::canonicalize_path_maybe_not_exists(
&FsSysTraitsAdapter::new_real(),
path,
)
}
/// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`.
@ -346,7 +126,7 @@ pub fn collect_specifiers(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(vendor_folder)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)?;
.collect_file_patterns(&FsSysTraitsAdapter::new_real(), files)?;
let mut collected_files_as_urls = collected_files
.iter()
.map(|f| specifier_from_file_path(f).unwrap())
@ -418,7 +198,13 @@ mod clone_dir_imp {
from: &std::path::Path,
to: &std::path::Path,
) -> Result<(), deno_core::error::AnyError> {
if let Err(e) = super::hard_link_dir_recursive(from, to) {
use deno_runtime::deno_fs::FsSysTraitsAdapter;
if let Err(e) = deno_npm_cache::hard_link_dir_recursive(
&FsSysTraitsAdapter::new_real(),
from,
to,
) {
log::debug!("Failed to hard link dir {:?} to {:?}: {}", from, to, e);
super::copy_dir_recursive(from, to)?;
}
@ -465,84 +251,6 @@ pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
Ok(())
}
/// Hardlinks the files in one directory to another directory.
///
/// Note: Does not handle symlinks.
pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> {
std::fs::create_dir_all(to)
.with_context(|| format!("Creating {}", to.display()))?;
let read_dir = std::fs::read_dir(from)
.with_context(|| format!("Reading {}", from.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let new_from = from.join(entry.file_name());
let new_to = to.join(entry.file_name());
if file_type.is_dir() {
hard_link_dir_recursive(&new_from, &new_to).with_context(|| {
format!("Dir {} to {}", new_from.display(), new_to.display())
})?;
} else if file_type.is_file() {
// note: chance for race conditions here between attempting to create,
// then removing, then attempting to create. There doesn't seem to be
// a way to hard link with overwriting in Rust, but maybe there is some
// way with platform specific code. The workaround here is to handle
// scenarios where something else might create or remove files.
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
if err.kind() == ErrorKind::AlreadyExists {
if let Err(err) = std::fs::remove_file(&new_to) {
if err.kind() == ErrorKind::NotFound {
// Assume another process/thread created this hard link to the file we are wanting
// to remove then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Removing file to hard link {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
// Always attempt to recreate the hardlink. In contention scenarios, the other process
// might have been killed or exited after removing the file, but before creating the hardlink
if let Err(err) = std::fs::hard_link(&new_from, &new_to) {
// Assume another process/thread created this hard link to the file we are wanting
// to now create then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
if err.kind() == ErrorKind::AlreadyExists {
std::thread::sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
}
}
Ok(())
}
pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), Error> {
let err_mapper = |err: Error, kind: Option<ErrorKind>| {
Error::new(

14
cli/util/path.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::fmt::Write;
use std::path::Path;
use std::path::PathBuf;
@ -52,19 +51,6 @@ pub fn get_extension(file_path: &Path) -> Option<String> {
.map(|e| e.to_lowercase());
}
pub fn get_atomic_file_path(file_path: &Path) -> PathBuf {
let rand = gen_rand_path_component();
let extension = format!("{rand}.tmp");
file_path.with_extension(extension)
}
fn gen_rand_path_component() -> String {
(0..4).fold(String::with_capacity(8), |mut output, _| {
write!(&mut output, "{:02x}", rand::random::<u8>()).unwrap();
output
})
}
/// TypeScript figures out the type of file based on the extension, but we take
/// other factors into account like the file headers. The hack here is to map the
/// specifier passed to TypeScript to a new specifier with the file extension.

2
ext/fs/Cargo.toml
View file

@ -25,10 +25,12 @@ deno_io.workspace = true
deno_path_util.workspace = true
deno_permissions.workspace = true
filetime.workspace = true
getrandom = "0.2"
libc.workspace = true
rand.workspace = true
rayon = "1.8.0"
serde.workspace = true
sys_traits.workspace = true
thiserror.workspace = true
[target.'cfg(unix)'.dependencies]

342
ext/fs/interface.rs
View file

@ -5,6 +5,8 @@ use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::time::Duration;
use std::time::SystemTime;
use serde::Deserialize;
use serde::Serialize;
@ -12,6 +14,8 @@ use serde::Serialize;
use deno_io::fs::File;
use deno_io::fs::FsResult;
use deno_io::fs::FsStat;
use sys_traits::FsFile;
use sys_traits::FsFileSetPermissions;
use crate::sync::MaybeSend;
use crate::sync::MaybeSync;
@ -71,7 +75,7 @@ pub enum FsFileType {
}
/// WARNING: This is part of the public JS Deno API.
#[derive(Serialize)]
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FsDirEntry {
pub name: String,
@ -100,6 +104,56 @@ impl<T> AccessCheckFn for T where
{
}
#[derive(Debug)]
pub struct FsStatSlim {
file_type: sys_traits::FileType,
modified: Result<SystemTime, std::io::Error>,
}
impl FsStatSlim {
pub fn from_std(metadata: &std::fs::Metadata) -> Self {
Self {
file_type: metadata.file_type().into(),
modified: metadata.modified(),
}
}
pub fn from_deno_fs_stat(data: &FsStat) -> Self {
FsStatSlim {
file_type: if data.is_file {
sys_traits::FileType::File
} else if data.is_directory {
sys_traits::FileType::Dir
} else if data.is_symlink {
sys_traits::FileType::Symlink
} else {
sys_traits::FileType::Unknown
},
modified: data
.mtime
.map(|ms| SystemTime::UNIX_EPOCH + Duration::from_millis(ms))
.ok_or_else(|| {
std::io::Error::new(std::io::ErrorKind::InvalidData, "No mtime")
}),
}
}
}
impl sys_traits::FsMetadataValue for FsStatSlim {
#[inline]
fn file_type(&self) -> sys_traits::FileType {
self.file_type
}
fn modified(&self) -> Result<SystemTime, std::io::Error> {
self
.modified
.as_ref()
.copied()
.map_err(|err| std::io::Error::new(err.kind(), err.to_string()))
}
}
pub type AccessCheckCb<'a> = &'a mut (dyn AccessCheckFn + 'a);
#[async_trait::async_trait(?Send)]
@ -361,3 +415,289 @@ fn string_from_utf8_lossy(buf: Vec<u8>) -> String {
Cow::Borrowed(_) => unsafe { String::from_utf8_unchecked(buf) },
}
}
// todo(dsherret): this is temporary. Instead of using the `FileSystem` trait implementation
// in the CLI, the CLI should instead create it's own file system using `sys_traits` traits
// then that can implement the `FileSystem` trait. Then this `FileSystem` trait can stay here
// for use only for `ext/fs` and not the entire CLI.
#[derive(Debug, Clone)]
pub struct FsSysTraitsAdapter(pub FileSystemRc);
impl FsSysTraitsAdapter {
pub fn new_real() -> Self {
Self(crate::sync::new_rc(crate::RealFs))
}
}
impl sys_traits::BaseFsHardLink for FsSysTraitsAdapter {
#[inline]
fn base_fs_hard_link(&self, src: &Path, dst: &Path) -> std::io::Result<()> {
self
.0
.link_sync(src, dst)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRead for FsSysTraitsAdapter {
#[inline]
fn base_fs_read(&self, path: &Path) -> std::io::Result<Cow<'static, [u8]>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
}
#[derive(Debug)]
pub struct FsSysTraitsAdapterReadDirEntry {
path: PathBuf,
entry: FsDirEntry,
}
impl sys_traits::FsDirEntry for FsSysTraitsAdapterReadDirEntry {
type Metadata = FsStatSlim;
fn file_name(&self) -> Cow<std::ffi::OsStr> {
Cow::Borrowed(self.entry.name.as_ref())
}
fn file_type(&self) -> std::io::Result<sys_traits::FileType> {
if self.entry.is_file {
Ok(sys_traits::FileType::File)
} else if self.entry.is_directory {
Ok(sys_traits::FileType::Dir)
} else if self.entry.is_symlink {
Ok(sys_traits::FileType::Symlink)
} else {
Ok(sys_traits::FileType::Unknown)
}
}
fn metadata(&self) -> std::io::Result<Self::Metadata> {
Ok(FsStatSlim {
file_type: self.file_type().unwrap(),
modified: Err(std::io::Error::new(
std::io::ErrorKind::Other,
"not supported",
)),
})
}
fn path(&self) -> Cow<Path> {
Cow::Borrowed(&self.path)
}
}
impl sys_traits::BaseFsReadDir for FsSysTraitsAdapter {
type ReadDirEntry = FsSysTraitsAdapterReadDirEntry;
fn base_fs_read_dir(
&self,
path: &Path,
) -> std::io::Result<
Box<dyn Iterator<Item = std::io::Result<Self::ReadDirEntry>>>,
> {
// todo(dsherret): needs to actually be iterable and not allocate a vector
let entries = self
.0
.read_dir_sync(path)
.map_err(|err| err.into_io_error())?;
let parent_dir = path.to_path_buf();
Ok(Box::new(entries.into_iter().map(move |entry| {
Ok(FsSysTraitsAdapterReadDirEntry {
path: parent_dir.join(&entry.name),
entry,
})
})))
}
}
impl sys_traits::BaseFsCanonicalize for FsSysTraitsAdapter {
#[inline]
fn base_fs_canonicalize(&self, path: &Path) -> std::io::Result<PathBuf> {
self
.0
.realpath_sync(path)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsMetadata for FsSysTraitsAdapter {
type Metadata = FsStatSlim;
#[inline]
fn base_fs_metadata(&self, path: &Path) -> std::io::Result<Self::Metadata> {
self
.0
.stat_sync(path)
.map(|data| FsStatSlim::from_deno_fs_stat(&data))
.map_err(|err| err.into_io_error())
}
#[inline]
fn base_fs_symlink_metadata(
&self,
path: &Path,
) -> std::io::Result<Self::Metadata> {
self
.0
.lstat_sync(path)
.map(|data| FsStatSlim::from_deno_fs_stat(&data))
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsCreateDir for FsSysTraitsAdapter {
#[inline]
fn base_fs_create_dir(
&self,
path: &Path,
options: &sys_traits::CreateDirOptions,
) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, options.recursive, options.mode)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRemoveFile for FsSysTraitsAdapter {
#[inline]
fn base_fs_remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.0
.remove_sync(path, false)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsRename for FsSysTraitsAdapter {
#[inline]
fn base_fs_rename(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self
.0
.rename_sync(from, to)
.map_err(|err| err.into_io_error())
}
}
pub struct FsFileAdapter(pub Rc<dyn File>);
impl FsFile for FsFileAdapter {}
impl FsFileSetPermissions for FsFileAdapter {
#[inline]
fn fs_file_set_permissions(&mut self, mode: u32) -> std::io::Result<()> {
if cfg!(windows) {
Ok(()) // ignore
} else {
self
.0
.clone()
.chmod_sync(mode)
.map_err(|err| err.into_io_error())
}
}
}
impl std::io::Read for FsFileAdapter {
#[inline]
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self
.0
.clone()
.read_sync(buf)
.map_err(|err| err.into_io_error())
}
}
impl std::io::Seek for FsFileAdapter {
fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result<u64> {
self
.0
.clone()
.seek_sync(pos)
.map_err(|err| err.into_io_error())
}
}
impl std::io::Write for FsFileAdapter {
#[inline]
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self
.0
.clone()
.write_sync(buf)
.map_err(|err| err.into_io_error())
}
#[inline]
fn flush(&mut self) -> std::io::Result<()> {
self
.0
.clone()
.sync_sync()
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::BaseFsOpen for FsSysTraitsAdapter {
type File = FsFileAdapter;
fn base_fs_open(
&self,
path: &Path,
options: &sys_traits::OpenOptions,
) -> std::io::Result<Self::File> {
self
.0
.open_sync(
path,
OpenOptions {
read: options.read,
write: options.write,
create: options.create,
truncate: options.truncate,
append: options.append,
create_new: options.create_new,
mode: options.mode,
},
None,
)
.map(FsFileAdapter)
.map_err(|err| err.into_io_error())
}
}
impl sys_traits::SystemRandom for FsSysTraitsAdapter {
#[inline]
fn sys_random(&self, buf: &mut [u8]) -> std::io::Result<()> {
getrandom::getrandom(buf).map_err(|err| {
std::io::Error::new(std::io::ErrorKind::Other, err.to_string())
})
}
}
impl sys_traits::SystemTimeNow for FsSysTraitsAdapter {
#[inline]
fn sys_time_now(&self) -> SystemTime {
SystemTime::now()
}
}
impl sys_traits::ThreadSleep for FsSysTraitsAdapter {
#[inline]
fn thread_sleep(&self, dur: Duration) {
std::thread::sleep(dur);
}
}
impl sys_traits::BaseEnvVar for FsSysTraitsAdapter {
fn base_env_var_os(
&self,
key: &std::ffi::OsStr,
) -> Option<std::ffi::OsString> {
std::env::var_os(key)
}
}

2
ext/fs/lib.rs
View file

@ -13,6 +13,8 @@ pub use crate::interface::FileSystem;
pub use crate::interface::FileSystemRc;
pub use crate::interface::FsDirEntry;
pub use crate::interface::FsFileType;
pub use crate::interface::FsStatSlim;
pub use crate::interface::FsSysTraitsAdapter;
pub use crate::interface::OpenOptions;
pub use crate::ops::FsOpsError;
pub use crate::ops::FsOpsErrorKind;

6
ext/fs/sync.rs
View file

@ -21,3 +21,9 @@ mod inner {
pub trait MaybeSend {}
impl<T> MaybeSend for T where T: ?Sized {}
}
#[allow(clippy::disallowed_types)]
#[inline]
pub fn new_rc<T>(value: T) -> MaybeArc<T> {
MaybeArc::new(value)
}

96
ext/node/lib.rs
View file

@ -14,7 +14,9 @@ use deno_core::url::Url;
#[allow(unused_imports)]
use deno_core::v8;
use deno_core::v8::ExternalReference;
use deno_fs::FsSysTraitsAdapter;
use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::IsBuiltInNodeModuleChecker;
use node_resolver::NpmPackageFolderResolverRc;
use once_cell::sync::Lazy;
@ -807,92 +809,28 @@ deno_core::extension!(deno_node,
},
);
pub type NodeResolver = node_resolver::NodeResolver<DenoFsNodeResolverEnv>;
#[allow(clippy::disallowed_types)]
pub type NodeResolverRc =
deno_fs::sync::MaybeArc<node_resolver::NodeResolver<DenoFsNodeResolverEnv>>;
pub type PackageJsonResolver =
node_resolver::PackageJsonResolver<DenoFsNodeResolverEnv>;
#[allow(clippy::disallowed_types)]
pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc<
node_resolver::PackageJsonResolver<DenoFsNodeResolverEnv>,
>;
#[derive(Debug)]
pub struct DenoFsNodeResolverEnv {
fs: deno_fs::FileSystemRc,
}
pub struct RealIsBuiltInNodeModuleChecker;
impl DenoFsNodeResolverEnv {
pub fn new(fs: deno_fs::FileSystemRc) -> Self {
Self { fs }
}
}
impl node_resolver::env::NodeResolverEnv for DenoFsNodeResolverEnv {
impl IsBuiltInNodeModuleChecker for RealIsBuiltInNodeModuleChecker {
#[inline]
fn is_builtin_node_module(&self, specifier: &str) -> bool {
is_builtin_node_module(specifier)
}
fn realpath_sync(
&self,
path: &std::path::Path,
) -> std::io::Result<std::path::PathBuf> {
self
.fs
.realpath_sync(path)
.map_err(|err| err.into_io_error())
}
fn stat_sync(
&self,
path: &std::path::Path,
) -> std::io::Result<node_resolver::env::NodeResolverFsStat> {
self
.fs
.stat_sync(path)
.map(|stat| node_resolver::env::NodeResolverFsStat {
is_file: stat.is_file,
is_dir: stat.is_directory,
is_symlink: stat.is_symlink,
})
.map_err(|err| err.into_io_error())
}
fn exists_sync(&self, path: &std::path::Path) -> bool {
self.fs.exists_sync(path)
}
fn pkg_json_fs(&self) -> &dyn deno_package_json::fs::DenoPkgJsonFs {
self
}
}
impl deno_package_json::fs::DenoPkgJsonFs for DenoFsNodeResolverEnv {
fn read_to_string_lossy(
&self,
path: &std::path::Path,
) -> Result<Cow<'static, str>, std::io::Error> {
self
.fs
.read_text_file_lossy_sync(path, None)
.map_err(|err| err.into_io_error())
}
}
pub struct DenoPkgJsonFsAdapter<'a>(pub &'a dyn deno_fs::FileSystem);
impl<'a> deno_package_json::fs::DenoPkgJsonFs for DenoPkgJsonFsAdapter<'a> {
fn read_to_string_lossy(
&self,
path: &Path,
) -> Result<Cow<'static, str>, std::io::Error> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|err| err.into_io_error())
}
}
pub type NodeResolver = node_resolver::NodeResolver<
RealIsBuiltInNodeModuleChecker,
FsSysTraitsAdapter,
>;
#[allow(clippy::disallowed_types)]
pub type NodeResolverRc = deno_fs::sync::MaybeArc<NodeResolver>;
pub type PackageJsonResolver =
node_resolver::PackageJsonResolver<FsSysTraitsAdapter>;
#[allow(clippy::disallowed_types)]
pub type PackageJsonResolverRc = deno_fs::sync::MaybeArc<
node_resolver::PackageJsonResolver<FsSysTraitsAdapter>,
>;
pub fn create_host_defined_options<'s>(
scope: &mut v8::HandleScope<'s>,

1
resolvers/deno/Cargo.toml
View file

@ -29,6 +29,7 @@ deno_path_util.workspace = true
deno_semver.workspace = true
node_resolver.workspace = true
node_resolver.features = ["sync"]
sys_traits.workspace = true
thiserror.workspace = true
url.workspace = true

21
resolvers/deno/cjs.rs
View file

@ -1,29 +1,30 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::sync::MaybeDashMap;
use deno_media_type::MediaType;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::ClosestPkgJsonError;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::PackageJsonResolverRc;
use node_resolver::ResolutionMode;
use sys_traits::FsRead;
use url::Url;
use crate::sync::MaybeDashMap;
/// Keeps track of what module specifiers were resolved as CJS.
///
/// Modules that are `.js`, `.ts`, `.jsx`, and `tsx` are only known to
/// be CJS or ESM after they're loaded based on their contents. So these
/// files will be "maybe CJS" until they're loaded.
#[derive(Debug)]
pub struct CjsTracker<TEnv: NodeResolverEnv> {
is_cjs_resolver: IsCjsResolver<TEnv>,
pub struct CjsTracker<TSys: FsRead> {
is_cjs_resolver: IsCjsResolver<TSys>,
known: MaybeDashMap<Url, ResolutionMode>,
}
impl<TEnv: NodeResolverEnv> CjsTracker<TEnv> {
impl<TSys: FsRead> CjsTracker<TSys> {
pub fn new(
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
mode: IsCjsResolutionMode,
) -> Self {
Self {
@ -124,16 +125,16 @@ pub enum IsCjsResolutionMode {
/// Resolves whether a module is CJS or ESM.
#[derive(Debug)]
pub struct IsCjsResolver<TEnv: NodeResolverEnv> {
pub struct IsCjsResolver<TSys: FsRead> {
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
mode: IsCjsResolutionMode,
}
impl<TEnv: NodeResolverEnv> IsCjsResolver<TEnv> {
impl<TSys: FsRead> IsCjsResolver<TSys> {
pub fn new(
in_npm_pkg_checker: InNpmPackageCheckerRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
mode: IsCjsResolutionMode,
) -> Self {
Self {

22
resolvers/deno/fs.rs
View file

@ -1,22 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::borrow::Cow;
use std::path::Path;
use std::path::PathBuf;
pub struct DirEntry {
pub name: String,
pub is_file: bool,
pub is_directory: bool,
}
pub trait DenoResolverFs {
fn read_to_string_lossy(
&self,
path: &Path,
) -> std::io::Result<Cow<'static, str>>;
fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf>;
fn exists_sync(&self, path: &Path) -> bool;
fn is_dir_sync(&self, path: &Path) -> bool;
fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result<Vec<DirEntry>>;
}

41
resolvers/deno/lib.rs
View file

@ -14,11 +14,10 @@ use deno_config::workspace::WorkspaceResolver;
use deno_package_json::PackageJsonDepValue;
use deno_package_json::PackageJsonDepValueParseError;
use deno_semver::npm::NpmPackageReqReference;
use fs::DenoResolverFs;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::NodeResolveError;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::IsBuiltInNodeModuleChecker;
use node_resolver::NodeResolution;
use node_resolver::NodeResolutionKind;
use node_resolver::NodeResolverRc;
@ -32,11 +31,14 @@ use npm::ResolveReqWithSubPathErrorKind;
use sloppy_imports::SloppyImportResolverFs;
use sloppy_imports::SloppyImportsResolutionKind;
use sloppy_imports::SloppyImportsResolverRc;
use sys_traits::FsCanonicalize;
use sys_traits::FsMetadata;
use sys_traits::FsRead;
use sys_traits::FsReadDir;
use thiserror::Error;
use url::Url;
pub mod cjs;
pub mod fs;
pub mod npm;
pub mod sloppy_imports;
mod sync;
@ -80,22 +82,22 @@ pub enum DenoResolveErrorKind {
#[derive(Debug)]
pub struct NodeAndNpmReqResolver<
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> {
pub node_resolver: NodeResolverRc<TNodeResolverEnv>,
pub npm_req_resolver: NpmReqResolverRc<Fs, TNodeResolverEnv>,
pub node_resolver: NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
pub npm_req_resolver: NpmReqResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
}
pub struct DenoResolverOptions<
'a,
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSloppyImportResolverFs: SloppyImportResolverFs,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> {
pub in_npm_pkg_checker: InNpmPackageCheckerRc,
pub node_and_req_resolver:
Option<NodeAndNpmReqResolver<Fs, TNodeResolverEnv>>,
Option<NodeAndNpmReqResolver<TIsBuiltInNodeModuleChecker, TSys>>,
pub sloppy_imports_resolver:
Option<SloppyImportsResolverRc<TSloppyImportResolverFs>>,
pub workspace_resolver: WorkspaceResolverRc,
@ -110,12 +112,13 @@ pub struct DenoResolverOptions<
/// import map, JSX settings.
#[derive(Debug)]
pub struct DenoResolver<
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSloppyImportResolverFs: SloppyImportResolverFs,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> {
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_and_npm_resolver: Option<NodeAndNpmReqResolver<Fs, TNodeResolverEnv>>,
node_and_npm_resolver:
Option<NodeAndNpmReqResolver<TIsBuiltInNodeModuleChecker, TSys>>,
sloppy_imports_resolver:
Option<SloppyImportsResolverRc<TSloppyImportResolverFs>>,
workspace_resolver: WorkspaceResolverRc,
@ -124,13 +127,17 @@ pub struct DenoResolver<
}
impl<
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSloppyImportResolverFs: SloppyImportResolverFs,
> DenoResolver<Fs, TNodeResolverEnv, TSloppyImportResolverFs>
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> DenoResolver<TIsBuiltInNodeModuleChecker, TSloppyImportResolverFs, TSys>
{
pub fn new(
options: DenoResolverOptions<Fs, TNodeResolverEnv, TSloppyImportResolverFs>,
options: DenoResolverOptions<
TIsBuiltInNodeModuleChecker,
TSloppyImportResolverFs,
TSys,
>,
) -> Self {
Self {
in_npm_pkg_checker: options.in_npm_pkg_checker,

117
resolvers/deno/npm/byonm.rs
View file

@ -11,7 +11,6 @@ use deno_path_util::url_to_file_path;
use deno_semver::package::PackageReq;
use deno_semver::StackString;
use deno_semver::Version;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::PackageFolderResolveError;
use node_resolver::errors::PackageFolderResolveIoError;
use node_resolver::errors::PackageJsonLoadError;
@ -19,11 +18,14 @@ use node_resolver::errors::PackageNotFoundError;
use node_resolver::InNpmPackageChecker;
use node_resolver::NpmPackageFolderResolver;
use node_resolver::PackageJsonResolverRc;
use sys_traits::FsCanonicalize;
use sys_traits::FsDirEntry;
use sys_traits::FsMetadata;
use sys_traits::FsRead;
use sys_traits::FsReadDir;
use thiserror::Error;
use url::Url;
use crate::fs::DenoResolverFs;
use super::local::normalize_pkg_name_for_node_modules_deno_folder;
use super::CliNpmReqResolver;
use super::ResolvePkgFolderFromDenoReqError;
@ -40,44 +42,45 @@ pub enum ByonmResolvePkgFolderFromDenoReqError {
Io(#[from] std::io::Error),
}
pub struct ByonmNpmResolverCreateOptions<
Fs: DenoResolverFs,
TEnv: NodeResolverEnv,
> {
pub struct ByonmNpmResolverCreateOptions<TSys: FsRead> {
// todo(dsherret): investigate removing this
pub root_node_modules_dir: Option<PathBuf>,
pub fs: Fs,
pub pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pub sys: TSys,
pub pkg_json_resolver: PackageJsonResolverRc<TSys>,
}
#[allow(clippy::disallowed_types)]
pub type ByonmNpmResolverRc<Fs, TEnv> =
crate::sync::MaybeArc<ByonmNpmResolver<Fs, TEnv>>;
pub type ByonmNpmResolverRc<TSys> =
crate::sync::MaybeArc<ByonmNpmResolver<TSys>>;
#[derive(Debug)]
pub struct ByonmNpmResolver<Fs: DenoResolverFs, TEnv: NodeResolverEnv> {
fs: Fs,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pub struct ByonmNpmResolver<
TSys: FsCanonicalize + FsRead + FsMetadata + FsReadDir,
> {
sys: TSys,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
root_node_modules_dir: Option<PathBuf>,
}
impl<Fs: DenoResolverFs + Clone, TEnv: NodeResolverEnv> Clone
for ByonmNpmResolver<Fs, TEnv>
impl<TSys: Clone + FsCanonicalize + FsRead + FsMetadata + FsReadDir> Clone
for ByonmNpmResolver<TSys>
{
fn clone(&self) -> Self {
Self {
fs: self.fs.clone(),
sys: self.sys.clone(),
pkg_json_resolver: self.pkg_json_resolver.clone(),
root_node_modules_dir: self.root_node_modules_dir.clone(),
}
}
}
impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
pub fn new(options: ByonmNpmResolverCreateOptions<Fs, TEnv>) -> Self {
impl<TSys: FsCanonicalize + FsRead + FsMetadata + FsReadDir>
ByonmNpmResolver<TSys>
{
pub fn new(options: ByonmNpmResolverCreateOptions<TSys>) -> Self {
Self {
root_node_modules_dir: options.root_node_modules_dir,
fs: options.fs,
sys: options.sys,
pkg_json_resolver: options.pkg_json_resolver,
}
}
@ -129,19 +132,20 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
req: &PackageReq,
referrer: &Url,
) -> Result<PathBuf, ByonmResolvePkgFolderFromDenoReqError> {
fn node_resolve_dir<Fs: DenoResolverFs>(
fs: &Fs,
fn node_resolve_dir<TSys: FsCanonicalize + FsMetadata>(
sys: &TSys,
alias: &str,
start_dir: &Path,
) -> std::io::Result<Option<PathBuf>> {
for ancestor in start_dir.ancestors() {
let node_modules_folder = ancestor.join("node_modules");
let sub_dir = join_package_name(&node_modules_folder, alias);
if fs.is_dir_sync(&sub_dir) {
return Ok(Some(deno_path_util::canonicalize_path_maybe_not_exists(
&sub_dir,
&|path| fs.realpath_sync(path),
)?));
if sys.fs_is_dir_no_err(&sub_dir) {
return Ok(Some(
deno_path_util::fs::canonicalize_path_maybe_not_exists(
sys, &sub_dir,
)?,
));
}
}
Ok(None)
@ -154,7 +158,7 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
Some((pkg_json, alias)) => {
// now try node resolution
if let Some(resolved) =
node_resolve_dir(&self.fs, &alias, pkg_json.dir_path())?
node_resolve_dir(&self.sys, &alias, pkg_json.dir_path())?
{
return Ok(resolved);
}
@ -298,7 +302,7 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
// now check if node_modules/.deno/ matches this constraint
let root_node_modules_dir = self.root_node_modules_dir.as_ref()?;
let node_modules_deno_dir = root_node_modules_dir.join(".deno");
let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else {
let Ok(entries) = self.sys.fs_read_dir(&node_modules_deno_dir) else {
return None;
};
let search_prefix = format!(
@ -311,10 +315,17 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
// - @denotest+add@1.0.0
// - @denotest+add@1.0.0_1
for entry in entries {
if !entry.is_directory {
let Ok(entry) = entry else {
continue;
};
let Ok(file_type) = entry.file_type() else {
continue;
};
if !file_type.is_dir() {
continue;
}
let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix)
let entry_name = entry.file_name().to_string_lossy().into_owned();
let Some(version_and_copy_idx) = entry_name.strip_prefix(&search_prefix)
else {
continue;
};
@ -327,8 +338,8 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
};
if let Some(tag) = req.version_req.tag() {
let initialized_file =
node_modules_deno_dir.join(&entry.name).join(".initialized");
let Ok(contents) = self.fs.read_to_string_lossy(&initialized_file)
node_modules_deno_dir.join(&entry_name).join(".initialized");
let Ok(contents) = self.sys.fs_read_to_string_lossy(&initialized_file)
else {
continue;
};
@ -336,19 +347,19 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
if tags.any(|t| t == tag) {
if let Some((best_version_version, _)) = &best_version {
if version > *best_version_version {
best_version = Some((version, entry.name));
best_version = Some((version, entry_name));
}
} else {
best_version = Some((version, entry.name));
best_version = Some((version, entry_name));
}
}
} else if req.version_req.matches(&version) {
if let Some((best_version_version, _)) = &best_version {
if version > *best_version_version {
best_version = Some((version, entry.name));
best_version = Some((version, entry_name));
}
} else {
best_version = Some((version, entry.name));
best_version = Some((version, entry_name));
}
}
}
@ -363,9 +374,14 @@ impl<Fs: DenoResolverFs, TEnv: NodeResolverEnv> ByonmNpmResolver<Fs, TEnv> {
}
impl<
Fs: DenoResolverFs + Send + Sync + std::fmt::Debug,
TEnv: NodeResolverEnv,
> CliNpmReqResolver for ByonmNpmResolver<Fs, TEnv>
Sys: FsCanonicalize
+ FsMetadata
+ FsRead
+ FsReadDir
+ Send
+ Sync
+ std::fmt::Debug,
> CliNpmReqResolver for ByonmNpmResolver<Sys>
{
fn resolve_pkg_folder_from_deno_module_req(
&self,
@ -380,17 +396,22 @@ impl<
}
impl<
Fs: DenoResolverFs + Send + Sync + std::fmt::Debug,
TEnv: NodeResolverEnv,
> NpmPackageFolderResolver for ByonmNpmResolver<Fs, TEnv>
Sys: FsCanonicalize
+ FsMetadata
+ FsRead
+ FsReadDir
+ Send
+ Sync
+ std::fmt::Debug,
> NpmPackageFolderResolver for ByonmNpmResolver<Sys>
{
fn resolve_package_folder_from_package(
&self,
name: &str,
referrer: &Url,
) -> Result<PathBuf, PackageFolderResolveError> {
fn inner<Fs: DenoResolverFs>(
fs: &Fs,
fn inner<TSys: FsMetadata>(
sys: &TSys,
name: &str,
referrer: &Url,
) -> Result<PathBuf, PackageFolderResolveError> {
@ -407,7 +428,7 @@ impl<
};
let sub_dir = join_package_name(&node_modules_folder, name);
if fs.is_dir_sync(&sub_dir) {
if sys.fs_is_dir_no_err(&sub_dir) {
return Ok(sub_dir);
}
}
@ -423,8 +444,8 @@ impl<
)
}
let path = inner(&self.fs, name, referrer)?;
self.fs.realpath_sync(&path).map_err(|err| {
let path = inner(&self.sys, name, referrer)?;
self.sys.fs_canonicalize(&path).map_err(|err| {
PackageFolderResolveIoError {
package_name: name.to_string(),
referrer: referrer.clone(),

48
resolvers/deno/npm/mod.rs
View file

@ -6,7 +6,6 @@ use std::path::PathBuf;
use boxed_error::Boxed;
use deno_semver::npm::NpmPackageReqReference;
use deno_semver::package::PackageReq;
use node_resolver::env::NodeResolverEnv;
use node_resolver::errors::NodeResolveError;
use node_resolver::errors::NodeResolveErrorKind;
use node_resolver::errors::PackageFolderResolveErrorKind;
@ -15,15 +14,18 @@ use node_resolver::errors::PackageNotFoundError;
use node_resolver::errors::PackageResolveErrorKind;
use node_resolver::errors::PackageSubpathResolveError;
use node_resolver::InNpmPackageCheckerRc;
use node_resolver::IsBuiltInNodeModuleChecker;
use node_resolver::NodeResolution;
use node_resolver::NodeResolutionKind;
use node_resolver::NodeResolverRc;
use node_resolver::ResolutionMode;
use sys_traits::FsCanonicalize;
use sys_traits::FsMetadata;
use sys_traits::FsRead;
use sys_traits::FsReadDir;
use thiserror::Error;
use url::Url;
use crate::fs::DenoResolverFs;
pub use byonm::ByonmInNpmPackageChecker;
pub use byonm::ByonmNpmResolver;
pub use byonm::ByonmNpmResolverCreateOptions;
@ -95,40 +97,46 @@ pub trait CliNpmReqResolver: Debug + Send + Sync {
}
pub struct NpmReqResolverOptions<
Fs: DenoResolverFs,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> {
/// The resolver when "bring your own node_modules" is enabled where Deno
/// does not setup the node_modules directories automatically, but instead
/// uses what already exists on the file system.
pub byonm_resolver: Option<ByonmNpmResolverRc<Fs, TNodeResolverEnv>>,
pub fs: Fs,
pub byonm_resolver: Option<ByonmNpmResolverRc<TSys>>,
pub in_npm_pkg_checker: InNpmPackageCheckerRc,
pub node_resolver: NodeResolverRc<TNodeResolverEnv>,
pub node_resolver: NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
pub npm_req_resolver: CliNpmReqResolverRc,
pub sys: TSys,
}
#[allow(clippy::disallowed_types)]
pub type NpmReqResolverRc<Fs, TNodeResolverEnv> =
crate::sync::MaybeArc<NpmReqResolver<Fs, TNodeResolverEnv>>;
pub type NpmReqResolverRc<TIsBuiltInNodeModuleChecker, TSys> =
crate::sync::MaybeArc<NpmReqResolver<TIsBuiltInNodeModuleChecker, TSys>>;
#[derive(Debug)]
pub struct NpmReqResolver<Fs: DenoResolverFs, TNodeResolverEnv: NodeResolverEnv>
{
byonm_resolver: Option<ByonmNpmResolverRc<Fs, TNodeResolverEnv>>,
fs: Fs,
pub struct NpmReqResolver<
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> {
byonm_resolver: Option<ByonmNpmResolverRc<TSys>>,
sys: TSys,
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_resolver: NodeResolverRc<TNodeResolverEnv>,
node_resolver: NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
npm_resolver: CliNpmReqResolverRc,
}
impl<Fs: DenoResolverFs, TNodeResolverEnv: NodeResolverEnv>
NpmReqResolver<Fs, TNodeResolverEnv>
impl<
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead + FsReadDir,
> NpmReqResolver<TIsBuiltInNodeModuleChecker, TSys>
{
pub fn new(options: NpmReqResolverOptions<Fs, TNodeResolverEnv>) -> Self {
pub fn new(
options: NpmReqResolverOptions<TIsBuiltInNodeModuleChecker, TSys>,
) -> Self {
Self {
byonm_resolver: options.byonm_resolver,
fs: options.fs,
sys: options.sys,
in_npm_pkg_checker: options.in_npm_pkg_checker,
node_resolver: options.node_resolver,
npm_resolver: options.npm_req_resolver,
@ -175,7 +183,7 @@ impl<Fs: DenoResolverFs, TNodeResolverEnv: NodeResolverEnv>
Err(err) => {
if self.byonm_resolver.is_some() {
let package_json_path = package_folder.join("package.json");
if !self.fs.exists_sync(&package_json_path) {
if !self.sys.fs_exists_no_err(&package_json_path) {
return Err(
MissingPackageNodeModulesFolderError { package_json_path }.into(),
);

1
resolvers/node/Cargo.toml
View file

@ -29,6 +29,7 @@ once_cell.workspace = true
path-clean = "=0.1.0"
regex.workspace = true
serde_json.workspace = true
sys_traits.workspace = true
thiserror.workspace = true
tokio.workspace = true
url.workspace = true

41
resolvers/node/analyze.rs
View file

@ -16,11 +16,14 @@ use once_cell::sync::Lazy;
use anyhow::Context;
use anyhow::Error as AnyError;
use sys_traits::FsCanonicalize;
use sys_traits::FsMetadata;
use sys_traits::FsRead;
use url::Url;
use crate::env::NodeResolverEnv;
use crate::npm::InNpmPackageCheckerRc;
use crate::resolution::NodeResolverRc;
use crate::IsBuiltInNodeModuleChecker;
use crate::NodeResolutionKind;
use crate::NpmPackageFolderResolverRc;
use crate::PackageJsonResolverRc;
@ -60,34 +63,38 @@ pub trait CjsCodeAnalyzer {
pub struct NodeCodeTranslator<
TCjsCodeAnalyzer: CjsCodeAnalyzer,
TNodeResolverEnv: NodeResolverEnv,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead,
> {
cjs_code_analyzer: TCjsCodeAnalyzer,
env: TNodeResolverEnv,
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_resolver: NodeResolverRc<TNodeResolverEnv>,
node_resolver: NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
npm_resolver: NpmPackageFolderResolverRc,
pkg_json_resolver: PackageJsonResolverRc<TNodeResolverEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
sys: TSys,
}
impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv>
NodeCodeTranslator<TCjsCodeAnalyzer, TNodeResolverEnv>
impl<
TCjsCodeAnalyzer: CjsCodeAnalyzer,
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead,
> NodeCodeTranslator<TCjsCodeAnalyzer, TIsBuiltInNodeModuleChecker, TSys>
{
pub fn new(
cjs_code_analyzer: TCjsCodeAnalyzer,
env: TNodeResolverEnv,
in_npm_pkg_checker: InNpmPackageCheckerRc,
node_resolver: NodeResolverRc<TNodeResolverEnv>,
node_resolver: NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys>,
npm_resolver: NpmPackageFolderResolverRc,
pkg_json_resolver: PackageJsonResolverRc<TNodeResolverEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
sys: TSys,
) -> Self {
Self {
cjs_code_analyzer,
env,
in_npm_pkg_checker,
node_resolver,
npm_resolver,
pkg_json_resolver,
sys,
}
}
@ -366,7 +373,7 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv>
// old school
if package_subpath != "." {
let d = module_dir.join(package_subpath);
if self.env.is_dir_sync(&d) {
if self.sys.fs_is_dir_no_err(&d) {
// subdir might have a package.json that specifies the entrypoint
let package_json_path = d.join("package.json");
let maybe_package_json = self
@ -423,13 +430,13 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv>
referrer: &Path,
) -> Result<PathBuf, AnyError> {
let p = p.clean();
if self.env.exists_sync(&p) {
if self.sys.fs_exists_no_err(&p) {
let file_name = p.file_name().unwrap();
let p_js =
p.with_file_name(format!("{}.js", file_name.to_str().unwrap()));
if self.env.is_file_sync(&p_js) {
if self.sys.fs_is_file_no_err(&p_js) {
return Ok(p_js);
} else if self.env.is_dir_sync(&p) {
} else if self.sys.fs_is_dir_no_err(&p) {
return Ok(p.join("index.js"));
} else {
return Ok(p);
@ -438,14 +445,14 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv>
{
let p_js =
p.with_file_name(format!("{}.js", file_name.to_str().unwrap()));
if self.env.is_file_sync(&p_js) {
if self.sys.fs_is_file_no_err(&p_js) {
return Ok(p_js);
}
}
{
let p_json =
p.with_file_name(format!("{}.json", file_name.to_str().unwrap()));
if self.env.is_file_sync(&p_json) {
if self.sys.fs_is_file_no_err(&p_json) {
return Ok(p_json);
}
}

39
resolvers/node/env.rs
View file

@ -1,39 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use crate::sync::MaybeSend;
use crate::sync::MaybeSync;
pub struct NodeResolverFsStat {
pub is_file: bool,
pub is_dir: bool,
pub is_symlink: bool,
}
pub trait NodeResolverEnv: std::fmt::Debug + MaybeSend + MaybeSync {
fn is_builtin_node_module(&self, specifier: &str) -> bool;
fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf>;
fn stat_sync(&self, path: &Path) -> std::io::Result<NodeResolverFsStat>;
fn exists_sync(&self, path: &Path) -> bool;
fn is_file_sync(&self, path: &Path) -> bool {
self
.stat_sync(path)
.map(|stat| stat.is_file)
.unwrap_or(false)
}
fn is_dir_sync(&self, path: &Path) -> bool {
self
.stat_sync(path)
.map(|stat| stat.is_dir)
.unwrap_or(false)
}
fn pkg_json_fs(&self) -> &dyn deno_package_json::fs::DenoPkgJsonFs;
}

2
resolvers/node/lib.rs
View file

@ -4,7 +4,6 @@
#![deny(clippy::print_stdout)]
pub mod analyze;
pub mod env;
pub mod errors;
mod npm;
mod package_json;
@ -23,6 +22,7 @@ pub use package_json::PackageJsonThreadLocalCache;
pub use path::PathClean;
pub use resolution::parse_npm_pkg_name;
pub use resolution::resolve_specifier_into_node_modules;
pub use resolution::IsBuiltInNodeModuleChecker;
pub use resolution::NodeResolution;
pub use resolution::NodeResolutionKind;
pub use resolution::NodeResolver;

25
resolvers/node/package_json.rs
View file

@ -1,15 +1,16 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_package_json::PackageJson;
use deno_package_json::PackageJsonRc;
use std::cell::RefCell;
use std::collections::HashMap;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use deno_package_json::PackageJson;
use deno_package_json::PackageJsonRc;
use sys_traits::FsRead;
use url::Url;
use crate::env::NodeResolverEnv;
use crate::errors::ClosestPkgJsonError;
use crate::errors::PackageJsonLoadError;
@ -38,17 +39,17 @@ impl deno_package_json::PackageJsonCache for PackageJsonThreadLocalCache {
}
#[allow(clippy::disallowed_types)]
pub type PackageJsonResolverRc<TEnv> =
crate::sync::MaybeArc<PackageJsonResolver<TEnv>>;
pub type PackageJsonResolverRc<TSys> =
crate::sync::MaybeArc<PackageJsonResolver<TSys>>;
#[derive(Debug)]
pub struct PackageJsonResolver<TEnv: NodeResolverEnv> {
env: TEnv,
pub struct PackageJsonResolver<TSys: FsRead> {
sys: TSys,
}
impl<TEnv: NodeResolverEnv> PackageJsonResolver<TEnv> {
pub fn new(env: TEnv) -> Self {
Self { env }
impl<TSys: FsRead> PackageJsonResolver<TSys> {
pub fn new(sys: TSys) -> Self {
Self { sys }
}
pub fn get_closest_package_json(
@ -81,9 +82,9 @@ impl<TEnv: NodeResolverEnv> PackageJsonResolver<TEnv> {
path: &Path,
) -> Result<Option<PackageJsonRc>, PackageJsonLoadError> {
let result = PackageJson::load_from_path(
path,
self.env.pkg_json_fs(),
&self.sys,
Some(&PackageJsonThreadLocalCache),
path,
);
match result {
Ok(pkg_json) => Ok(Some(pkg_json)),

111
resolvers/node/resolution.rs
View file

@ -9,9 +9,13 @@ use anyhow::Error as AnyError;
use deno_path_util::url_from_file_path;
use serde_json::Map;
use serde_json::Value;
use sys_traits::FileType;
use sys_traits::FsCanonicalize;
use sys_traits::FsMetadata;
use sys_traits::FsMetadataValue;
use sys_traits::FsRead;
use url::Url;
use crate::env::NodeResolverEnv;
use crate::errors;
use crate::errors::DataUrlReferrerError;
use crate::errors::FinalizeResolutionError;
@ -98,29 +102,44 @@ impl NodeResolution {
}
}
#[allow(clippy::disallowed_types)]
pub type NodeResolverRc<TEnv> = crate::sync::MaybeArc<NodeResolver<TEnv>>;
#[derive(Debug)]
pub struct NodeResolver<TEnv: NodeResolverEnv> {
env: TEnv,
in_npm_pkg_checker: InNpmPackageCheckerRc,
npm_pkg_folder_resolver: NpmPackageFolderResolverRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pub trait IsBuiltInNodeModuleChecker: std::fmt::Debug {
fn is_builtin_node_module(&self, specifier: &str) -> bool;
}
impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
#[allow(clippy::disallowed_types)]
pub type NodeResolverRc<TIsBuiltInNodeModuleChecker, TSys> =
crate::sync::MaybeArc<NodeResolver<TIsBuiltInNodeModuleChecker, TSys>>;
#[derive(Debug)]
pub struct NodeResolver<
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead,
> {
in_npm_pkg_checker: InNpmPackageCheckerRc,
is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker,
npm_pkg_folder_resolver: NpmPackageFolderResolverRc,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
sys: TSys,
}
impl<
TIsBuiltInNodeModuleChecker: IsBuiltInNodeModuleChecker,
TSys: FsCanonicalize + FsMetadata + FsRead,
> NodeResolver<TIsBuiltInNodeModuleChecker, TSys>
{
pub fn new(
env: TEnv,
in_npm_pkg_checker: InNpmPackageCheckerRc,
is_built_in_node_module_checker: TIsBuiltInNodeModuleChecker,
npm_pkg_folder_resolver: NpmPackageFolderResolverRc,
pkg_json_resolver: PackageJsonResolverRc<TEnv>,
pkg_json_resolver: PackageJsonResolverRc<TSys>,
sys: TSys,
) -> Self {
Self {
env,
in_npm_pkg_checker,
is_built_in_node_module_checker,
npm_pkg_folder_resolver,
pkg_json_resolver,
sys,
}
}
@ -140,7 +159,10 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
// Note: if we are here, then the referrer is an esm module
// TODO(bartlomieju): skipped "policy" part as we don't plan to support it
if self.env.is_builtin_node_module(specifier) {
if self
.is_built_in_node_module_checker
.is_builtin_node_module(specifier)
{
return Ok(NodeResolution::BuiltIn(specifier.to_string()));
}
@ -282,32 +304,25 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
p_str.to_string()
};
let (is_dir, is_file) = if let Ok(stats) = self.env.stat_sync(Path::new(&p))
{
(stats.is_dir, stats.is_file)
} else {
(false, false)
};
if is_dir {
return Err(
let maybe_file_type = self.sys.fs_metadata(p).map(|m| m.file_type());
match maybe_file_type {
Ok(FileType::Dir) => Err(
UnsupportedDirImportError {
dir_url: resolved.clone(),
maybe_referrer: maybe_referrer.map(ToOwned::to_owned),
}
.into(),
);
} else if !is_file {
return Err(
),
Ok(FileType::File) => Ok(resolved),
_ => Err(
ModuleNotFoundError {
specifier: resolved,
maybe_referrer: maybe_referrer.map(ToOwned::to_owned),
typ: "module",
}
.into(),
);
),
}
Ok(resolved)
}
pub fn resolve_package_subpath_from_deno_module(
@ -397,8 +412,8 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
maybe_referrer: Option<&Url>,
resolution_mode: ResolutionMode,
) -> Result<Url, TypesNotFoundError> {
fn probe_extensions<TEnv: NodeResolverEnv>(
fs: &TEnv,
fn probe_extensions<TSys: FsMetadata>(
sys: &TSys,
path: &Path,
lowercase_path: &str,
resolution_mode: ResolutionMode,
@ -407,20 +422,20 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
let mut searched_for_d_cts = false;
if lowercase_path.ends_with(".mjs") {
let d_mts_path = with_known_extension(path, "d.mts");
if fs.exists_sync(&d_mts_path) {
if sys.fs_exists_no_err(&d_mts_path) {
return Some(d_mts_path);
}
searched_for_d_mts = true;
} else if lowercase_path.ends_with(".cjs") {
let d_cts_path = with_known_extension(path, "d.cts");
if fs.exists_sync(&d_cts_path) {
if sys.fs_exists_no_err(&d_cts_path) {
return Some(d_cts_path);
}
searched_for_d_cts = true;
}
let dts_path = with_known_extension(path, "d.ts");
if fs.exists_sync(&dts_path) {
if sys.fs_exists_no_err(&dts_path) {
return Some(dts_path);
}
@ -434,7 +449,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
_ => None, // already searched above
};
if let Some(specific_dts_path) = specific_dts_path {
if fs.exists_sync(&specific_dts_path) {
if sys.fs_exists_no_err(&specific_dts_path) {
return Some(specific_dts_path);
}
}
@ -449,11 +464,11 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
return Ok(url_from_file_path(path).unwrap());
}
if let Some(path) =
probe_extensions(&self.env, path, &lowercase_path, resolution_mode)
probe_extensions(&self.sys, path, &lowercase_path, resolution_mode)
{
return Ok(url_from_file_path(&path).unwrap());
}
if self.env.is_dir_sync(path) {
if self.sys.fs_is_dir_no_err(path) {
let resolution_result = self.resolve_package_dir_subpath(
path,
/* sub path */ ".",
@ -467,7 +482,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
}
let index_path = path.join("index.js");
if let Some(path) = probe_extensions(
&self.env,
&self.sys,
&index_path,
&index_path.to_string_lossy().to_lowercase(),
resolution_mode,
@ -671,7 +686,10 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
return match result {
Ok(url) => Ok(url),
Err(err) => {
if self.env.is_builtin_node_module(target) {
if self
.is_built_in_node_module_checker
.is_builtin_node_module(target)
{
Ok(Url::parse(&format!("node:{}", target)).unwrap())
} else {
Err(err)
@ -1353,7 +1371,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
if let Some(main) = maybe_main {
let guess = package_json.path.parent().unwrap().join(main).clean();
if self.env.is_file_sync(&guess) {
if self.sys.fs_is_file_no_err(&guess) {
return Ok(url_from_file_path(&guess).unwrap());
}
@ -1382,7 +1400,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
.unwrap()
.join(format!("{main}{ending}"))
.clean();
if self.env.is_file_sync(&guess) {
if self.sys.fs_is_file_no_err(&guess) {
// TODO(bartlomieju): emitLegacyIndexDeprecation()
return Ok(url_from_file_path(&guess).unwrap());
}
@ -1417,7 +1435,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
};
for index_file_name in index_file_names {
let guess = directory.join(index_file_name).clean();
if self.env.is_file_sync(&guess) {
if self.sys.fs_is_file_no_err(&guess) {
// TODO(bartlomieju): emitLegacyIndexDeprecation()
return Ok(url_from_file_path(&guess).unwrap());
}
@ -1454,9 +1472,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> {
{
// Specifiers in the node_modules directory are canonicalized
// so canoncalize then check if it's in the node_modules directory.
let specifier = resolve_specifier_into_node_modules(specifier, &|path| {
self.env.realpath_sync(path)
});
let specifier = resolve_specifier_into_node_modules(&self.sys, specifier);
return Some(specifier);
}
@ -1717,16 +1733,15 @@ pub fn parse_npm_pkg_name(
/// not be fully resolved at the time deno_graph is analyzing it
/// because the node_modules folder might not exist at that time.
pub fn resolve_specifier_into_node_modules(
sys: &impl FsCanonicalize,
specifier: &Url,
canonicalize: &impl Fn(&Path) -> std::io::Result<PathBuf>,
) -> Url {
deno_path_util::url_to_file_path(specifier)
.ok()
// this path might not exist at the time the graph is being created
// because the node_modules folder might not yet exist
.and_then(|path| {
deno_path_util::canonicalize_path_maybe_not_exists(&path, canonicalize)
.ok()
deno_path_util::fs::canonicalize_path_maybe_not_exists(sys, &path).ok()
})
.and_then(|path| deno_path_util::url_from_file_path(&path).ok())
.unwrap_or_else(|| specifier.clone())

2
resolvers/npm_cache/Cargo.toml
View file

@ -25,6 +25,7 @@ boxed_error.workspace = true
deno_cache_dir.workspace = true
deno_error.workspace = true
deno_npm.workspace = true
deno_path_util.workspace = true
deno_semver.workspace = true
deno_unsync = { workspace = true, features = ["tokio"] }
faster-hex.workspace = true
@ -37,6 +38,7 @@ percent-encoding.workspace = true
rand.workspace = true
ring.workspace = true
serde_json.workspace = true
sys_traits.workspace = true
tar.workspace = true
tempfile = "3.4.0"
thiserror.workspace = true

99
resolvers/npm_cache/fs_util.rs Normal file
View file

@ -0,0 +1,99 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use anyhow::Context;
use anyhow::Error as AnyError;
use std::io::ErrorKind;
use std::path::Path;
use std::time::Duration;
use sys_traits::FsCreateDirAll;
use sys_traits::FsDirEntry;
use sys_traits::FsHardLink;
use sys_traits::FsReadDir;
use sys_traits::FsRemoveFile;
use sys_traits::ThreadSleep;
/// Hardlinks the files in one directory to another directory.
///
/// Note: Does not handle symlinks.
pub fn hard_link_dir_recursive<
TSys: FsCreateDirAll + FsHardLink + FsReadDir + FsRemoveFile + ThreadSleep,
>(
sys: &TSys,
from: &Path,
to: &Path,
) -> Result<(), AnyError> {
sys
.fs_create_dir_all(to)
.with_context(|| format!("Creating {}", to.display()))?;
let read_dir = sys
.fs_read_dir(from)
.with_context(|| format!("Reading {}", from.display()))?;
for entry in read_dir {
let entry = entry?;
let file_type = entry.file_type()?;
let new_from = from.join(entry.file_name());
let new_to = to.join(entry.file_name());
if file_type.is_dir() {
hard_link_dir_recursive(sys, &new_from, &new_to).with_context(|| {
format!("Dir {} to {}", new_from.display(), new_to.display())
})?;
} else if file_type.is_file() {
// note: chance for race conditions here between attempting to create,
// then removing, then attempting to create. There doesn't seem to be
// a way to hard link with overwriting in Rust, but maybe there is some
// way with platform specific code. The workaround here is to handle
// scenarios where something else might create or remove files.
if let Err(err) = sys.fs_hard_link(&new_from, &new_to) {
if err.kind() == ErrorKind::AlreadyExists {
if let Err(err) = sys.fs_remove_file(&new_to) {
if err.kind() == ErrorKind::NotFound {
// Assume another process/thread created this hard link to the file we are wanting
// to remove then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
sys.thread_sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Removing file to hard link {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
// Always attempt to recreate the hardlink. In contention scenarios, the other process
// might have been killed or exited after removing the file, but before creating the hardlink
if let Err(err) = sys.fs_hard_link(&new_from, &new_to) {
// Assume another process/thread created this hard link to the file we are wanting
// to now create then sleep a little bit to let the other process/thread move ahead
// faster to reduce contention.
if err.kind() == ErrorKind::AlreadyExists {
sys.thread_sleep(Duration::from_millis(10));
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
} else {
return Err(err).with_context(|| {
format!(
"Hard linking {} to {}",
new_from.display(),
new_to.display()
)
});
}
}
}
}
Ok(())
}

76
resolvers/npm_cache/lib.rs
View file

@ -14,6 +14,7 @@ use deno_cache_dir::npm::NpmCacheDir;
use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::NpmPackageCacheFolderId;
use deno_path_util::fs::atomic_write_file_with_retries;
use deno_semver::package::PackageNv;
use deno_semver::StackString;
use deno_semver::Version;
@ -21,13 +22,24 @@ use http::HeaderName;
use http::HeaderValue;
use http::StatusCode;
use parking_lot::Mutex;
use sys_traits::FsCreateDirAll;
use sys_traits::FsHardLink;
use sys_traits::FsMetadata;
use sys_traits::FsOpen;
use sys_traits::FsReadDir;
use sys_traits::FsRemoveFile;
use sys_traits::FsRename;
use sys_traits::SystemRandom;
use sys_traits::ThreadSleep;
use url::Url;
mod fs_util;
mod registry_info;
mod remote;
mod tarball;
mod tarball_extract;
pub use fs_util::hard_link_dir_recursive;
pub use registry_info::RegistryInfoProvider;
pub use tarball::TarballCache;
@ -55,18 +67,7 @@ impl std::fmt::Display for DownloadError {
}
#[async_trait::async_trait(?Send)]
pub trait NpmCacheEnv: Send + Sync + 'static {
fn exists(&self, path: &Path) -> bool;
fn hard_link_dir_recursive(
&self,
from: &Path,
to: &Path,
) -> Result<(), AnyError>;
fn atomic_write_file_with_retries(
&self,
file_path: &Path,
data: &[u8],
) -> std::io::Result<()>;
pub trait NpmCacheHttpClient: Send + Sync + 'static {
async fn download_with_retries_on_any_tokio_runtime(
&self,
url: Url,
@ -126,27 +127,48 @@ impl NpmCacheSetting {
/// Stores a single copy of npm packages in a cache.
#[derive(Debug)]
pub struct NpmCache<TEnv: NpmCacheEnv> {
env: Arc<TEnv>,
pub struct NpmCache<
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom,
> {
cache_dir: Arc<NpmCacheDir>,
sys: TSys,
cache_setting: NpmCacheSetting,
npmrc: Arc<ResolvedNpmRc>,
previously_reloaded_packages: Mutex<HashSet<PackageNv>>,
}
impl<TEnv: NpmCacheEnv> NpmCache<TEnv> {
impl<
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom,
> NpmCache<TSys>
{
pub fn new(
cache_dir: Arc<NpmCacheDir>,
sys: TSys,
cache_setting: NpmCacheSetting,
env: Arc<TEnv>,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
cache_dir,
sys,
cache_setting,
env,
previously_reloaded_packages: Default::default(),
npmrc,
previously_reloaded_packages: Default::default(),
}
}
@ -211,9 +233,11 @@ impl<TEnv: NpmCacheEnv> NpmCache<TEnv> {
// it seems Windows does an "AccessDenied" error when moving a
// directory with hard links, so that's why this solution is done
with_folder_sync_lock(&folder_id.nv, &package_folder, || {
self
.env
.hard_link_dir_recursive(&original_package_folder, &package_folder)
hard_link_dir_recursive(
&self.sys,
&original_package_folder,
&package_folder,
)
})?;
Ok(())
}
@ -290,9 +314,12 @@ impl<TEnv: NpmCacheEnv> NpmCache<TEnv> {
) -> Result<(), AnyError> {
let file_cache_path = self.get_registry_package_info_file_cache_path(name);
let file_text = serde_json::to_string(&package_info)?;
self
.env
.atomic_write_file_with_retries(&file_cache_path, file_text.as_bytes())?;
atomic_write_file_with_retries(
&self.sys,
&file_cache_path,
file_text.as_bytes(),
0o644,
)?;
Ok(())
}
@ -304,6 +331,7 @@ impl<TEnv: NpmCacheEnv> NpmCache<TEnv> {
const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock";
// todo(dsherret): use `sys` here instead of `std::fs`.
fn with_folder_sync_lock(
package: &PackageNv,
output_folder: &Path,

95
resolvers/npm_cache/registry_info.rs
View file

@ -18,12 +18,21 @@ use deno_unsync::sync::MultiRuntimeAsyncValueCreator;
use futures::future::LocalBoxFuture;
use futures::FutureExt;
use parking_lot::Mutex;
use sys_traits::FsCreateDirAll;
use sys_traits::FsHardLink;
use sys_traits::FsMetadata;
use sys_traits::FsOpen;
use sys_traits::FsReadDir;
use sys_traits::FsRemoveFile;
use sys_traits::FsRename;
use sys_traits::SystemRandom;
use sys_traits::ThreadSleep;
use thiserror::Error;
use url::Url;
use crate::remote::maybe_auth_header_for_npm_registry;
use crate::NpmCache;
use crate::NpmCacheEnv;
use crate::NpmCacheHttpClient;
use crate::NpmCacheSetting;
type LoadResult = Result<FutureResult, Arc<AnyError>>;
@ -122,25 +131,54 @@ impl MemoryCache {
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct RegistryInfoProvider<TEnv: NpmCacheEnv> {
pub struct RegistryInfoProvider<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
> {
// todo(#27198): remove this
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
npmrc: Arc<ResolvedNpmRc>,
force_reload_flag: AtomicFlag,
memory_cache: Mutex<MemoryCache>,
previously_loaded_packages: Mutex<HashSet<String>>,
}
impl<TEnv: NpmCacheEnv> RegistryInfoProvider<TEnv> {
impl<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
> RegistryInfoProvider<THttpClient, TSys>
{
pub fn new(
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
cache,
env,
http_client,
npmrc,
force_reload_flag: AtomicFlag::lowered(),
memory_cache: Default::default(),
@ -170,7 +208,9 @@ impl<TEnv: NpmCacheEnv> RegistryInfoProvider<TEnv> {
}
}
pub fn as_npm_registry_api(self: &Arc<Self>) -> NpmRegistryApiAdapter<TEnv> {
pub fn as_npm_registry_api(
self: &Arc<Self>,
) -> NpmRegistryApiAdapter<THttpClient, TSys> {
NpmRegistryApiAdapter(self.clone())
}
@ -341,7 +381,7 @@ impl<TEnv: NpmCacheEnv> RegistryInfoProvider<TEnv> {
downloader.previously_loaded_packages.lock().insert(name.to_string());
let maybe_bytes = downloader
.env
.http_client
.download_with_retries_on_any_tokio_runtime(
package_url,
maybe_auth_header,
@ -378,12 +418,39 @@ impl<TEnv: NpmCacheEnv> RegistryInfoProvider<TEnv> {
}
}
pub struct NpmRegistryApiAdapter<TEnv: NpmCacheEnv>(
Arc<RegistryInfoProvider<TEnv>>,
);
pub struct NpmRegistryApiAdapter<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
>(Arc<RegistryInfoProvider<THttpClient, TSys>>);
#[async_trait(?Send)]
impl<TEnv: NpmCacheEnv> NpmRegistryApi for NpmRegistryApiAdapter<TEnv> {
impl<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsReadDir
+ FsRemoveFile
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
> NpmRegistryApi for NpmRegistryApiAdapter<THttpClient, TSys>
{
async fn package_info(
&self,
name: &str,

61
resolvers/npm_cache/tarball.rs
View file

@ -15,13 +15,22 @@ use futures::future::LocalBoxFuture;
use futures::FutureExt;
use http::StatusCode;
use parking_lot::Mutex;
use sys_traits::FsCreateDirAll;
use sys_traits::FsHardLink;
use sys_traits::FsMetadata;
use sys_traits::FsOpen;
use sys_traits::FsReadDir;
use sys_traits::FsRemoveFile;
use sys_traits::FsRename;
use sys_traits::SystemRandom;
use sys_traits::ThreadSleep;
use url::Url;
use crate::remote::maybe_auth_header_for_npm_registry;
use crate::tarball_extract::verify_and_extract_tarball;
use crate::tarball_extract::TarballExtractionMode;
use crate::NpmCache;
use crate::NpmCacheEnv;
use crate::NpmCacheHttpClient;
use crate::NpmCacheSetting;
type LoadResult = Result<(), Arc<AnyError>>;
@ -42,22 +51,54 @@ enum MemoryCacheItem {
///
/// This is shared amongst all the workers.
#[derive(Debug)]
pub struct TarballCache<TEnv: NpmCacheEnv> {
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
pub struct TarballCache<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsRemoveFile
+ FsReadDir
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
> {
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
sys: TSys,
npmrc: Arc<ResolvedNpmRc>,
memory_cache: Mutex<HashMap<PackageNv, MemoryCacheItem>>,
}
impl<TEnv: NpmCacheEnv> TarballCache<TEnv> {
impl<
THttpClient: NpmCacheHttpClient,
TSys: FsCreateDirAll
+ FsHardLink
+ FsMetadata
+ FsOpen
+ FsRemoveFile
+ FsReadDir
+ FsRename
+ ThreadSleep
+ SystemRandom
+ Send
+ Sync
+ 'static,
> TarballCache<THttpClient, TSys>
{
pub fn new(
cache: Arc<NpmCache<TEnv>>,
env: Arc<TEnv>,
cache: Arc<NpmCache<TSys>>,
http_client: Arc<THttpClient>,
sys: TSys,
npmrc: Arc<ResolvedNpmRc>,
) -> Self {
Self {
cache,
env,
http_client,
sys,
npmrc,
memory_cache: Default::default(),
}
@ -131,7 +172,7 @@ impl<TEnv: NpmCacheEnv> TarballCache<TEnv> {
let package_folder =
tarball_cache.cache.package_folder_for_nv_and_url(&package_nv, registry_url);
let should_use_cache = tarball_cache.cache.should_use_cache_for_package(&package_nv);
let package_folder_exists = tarball_cache.env.exists(&package_folder);
let package_folder_exists = tarball_cache.sys.fs_exists_no_err(&package_folder);
if should_use_cache && package_folder_exists {
return Ok(());
} else if tarball_cache.cache.cache_setting() == &NpmCacheSetting::Only {
@ -156,7 +197,7 @@ impl<TEnv: NpmCacheEnv> TarballCache<TEnv> {
tarball_cache.npmrc.tarball_config(&tarball_uri);
let maybe_auth_header = maybe_registry_config.and_then(|c| maybe_auth_header_for_npm_registry(c).ok()?);
let result = tarball_cache.env
let result = tarball_cache.http_client
.download_with_retries_on_any_tokio_runtime(tarball_uri, maybe_auth_header)
.await;
let maybe_bytes = match result {

1
tests/Cargo.toml
View file

@ -60,6 +60,7 @@ pretty_assertions.workspace = true
regex.workspace = true
reqwest.workspace = true
serde.workspace = true
sys_traits = { workspace = true, features = ["real", "getrandom", "libc", "winapi"] }
test_util.workspace = true
tokio.workspace = true
tower-lsp.workspace = true

2
tests/integration/jsr_tests.rs
View file

@ -191,8 +191,8 @@ fn reload_info_not_found_cache_but_exists_remote() {
Url::parse(&format!("http://127.0.0.1:4250/{}/meta.json", package))
.unwrap();
let cache = deno_cache_dir::GlobalHttpCache::new(
sys_traits::impls::RealSys,
deno_dir.path().join("remote").to_path_buf(),
deno_cache_dir::TestRealDenoCacheEnv,
);
let entry = cache
.get(&cache.cache_item_key(&specifier).unwrap(), None)