foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-25 08:39:09 -05:00
Code Issues Packages 1 Wiki Activity

refactor: move NpmCacheDir to deno_cache_dir (#25916)

Browse source 
Part of the ongoing work to move more of Deno's resolution out of the
CLI crate (for use in Wasm and other things)

Includes:

* https://github.com/denoland/deno_cache_dir/pull/60
This commit is contained in:
David Sherret 2024-09-28 08:50:16 -04:00 committed by GitHub
parent fc739dc5eb
commit 1bb47805d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 267 additions and 411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
Cargo.lock generated
View file

@ -1348,10 +1348,11 @@ dependencies = [
[[package]]
name = "deno_cache_dir"
version = "0.11.1"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6df43311cb7703fa3242c282823a850e4c8d0c06b9527d8209b55bd695452ea5"
checksum = "87900cfcd07bdbf3597bc36b77da0c0e7b6c2e65213faa2ed43d9a1ec12bd31d"
dependencies = [
"base32",
"deno_media_type",
"indexmap",
"log",

2
Cargo.toml
View file

@ -104,7 +104,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde"
console_static_text = "=0.8.1"
data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.11.1"
deno_cache_dir = "=0.12.0"
deno_package_json = { version = "=0.1.1", default-features = false }
dlopen2 = "0.6.1"
ecb = "=0.1.2"

77
cli/cache/mod.rs vendored
View file

@ -10,6 +10,8 @@ use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
use crate::npm::CliNpmResolver;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
use crate::util::path::specifier_has_extension;
use deno_ast::MediaType;
@ -77,6 +79,14 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
atomic_write_file_with_retries(path, bytes, CACHE_PERM)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
crate::util::fs::canonicalize_path(path)
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(path)
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
@ -100,6 +110,73 @@ impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
}
}
#[derive(Debug, Clone)]
pub struct DenoCacheEnvFsAdapter<'a>(
pub &'a dyn deno_runtime::deno_fs::FileSystem,
);
impl<'a> deno_cache_dir::DenoCacheEnv for DenoCacheEnvFsAdapter<'a> {
fn read_file_bytes(&self, path: &Path) -> std::io::Result<Vec<u8>> {
self
.0
.read_file_sync(path, None)
.map_err(|err| err.into_io_error())
}
fn atomic_write_file(
&self,
path: &Path,
bytes: &[u8],
) -> std::io::Result<()> {
atomic_write_file_with_retries_and_fs(
&AtomicWriteFileFsAdapter {
fs: self.0,
write_mode: CACHE_PERM,
},
path,
bytes,
)
}
fn canonicalize_path(&self, path: &Path) -> std::io::Result<PathBuf> {
self.0.realpath_sync(path).map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, path: &Path) -> std::io::Result<()> {
self
.0
.mkdir_sync(path, true, None)
.map_err(|e| e.into_io_error())
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.0
.remove_sync(path, false)
.map_err(|e| e.into_io_error())
}
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
self
.0
.stat_sync(path)
.map(|stat| {
stat
.mtime
.map(|ts| SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(ts))
})
.map_err(|e| e.into_io_error())
}
fn is_file(&self, path: &Path) -> bool {
self.0.is_file_sync(path)
}
fn time_now(&self) -> SystemTime {
SystemTime::now()
}
}
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
pub type LocalLspHttpCache =

295
cli/npm/cache_dir.rs
View file

@ -1,295 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::url::Url;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use crate::util::fs::canonicalize_path;
use crate::util::path::root_url_to_safe_local_dirname;
/// The global cache directory of npm packages.
#[derive(Clone, Debug)]
pub struct NpmCacheDir {
root_dir: PathBuf,
// cached url representation of the root directory
root_dir_url: Url,
// A list of all registry that were discovered via `.npmrc` files
// turned into a safe directory names.
known_registries_dirnames: Vec<String>,
}
impl NpmCacheDir {
pub fn new(root_dir: PathBuf, known_registries_urls: Vec<Url>) -> Self {
fn try_get_canonicalized_root_dir(
root_dir: &Path,
) -> Result<PathBuf, AnyError> {
if !root_dir.exists() {
std::fs::create_dir_all(root_dir)
.with_context(|| format!("Error creating {}", root_dir.display()))?;
}
Ok(canonicalize_path(root_dir)?)
}
// this may fail on readonly file systems, so just ignore if so
let root_dir =
try_get_canonicalized_root_dir(&root_dir).unwrap_or(root_dir);
let root_dir_url = Url::from_directory_path(&root_dir).unwrap();
let known_registries_dirnames: Vec<_> = known_registries_urls
.into_iter()
.map(|url| {
root_url_to_safe_local_dirname(&url)
.to_string_lossy()
.replace('\\', "/")
})
.collect();
Self {
root_dir,
root_dir_url,
known_registries_dirnames,
}
}
pub fn root_dir(&self) -> &Path {
&self.root_dir
}
pub fn root_dir_url(&self) -> &Url {
&self.root_dir_url
}
pub fn package_folder_for_id(
&self,
folder_id: &NpmPackageCacheFolderId,
registry_url: &Url,
) -> PathBuf {
if folder_id.copy_index == 0 {
self.package_folder_for_nv(&folder_id.nv, registry_url)
} else {
self
.package_name_folder(&folder_id.nv.name, registry_url)
.join(format!("{}_{}", folder_id.nv.version, folder_id.copy_index))
}
}
pub fn package_folder_for_nv(
&self,
package: &PackageNv,
registry_url: &Url,
) -> PathBuf {
self
.package_name_folder(&package.name, registry_url)
.join(package.version.to_string())
}
pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf {
let mut dir = self.registry_folder(registry_url);
if name.to_lowercase() != name {
let encoded_name = mixed_case_package_name_encode(name);
// Using the encoded directory may have a collision with an actual package name
// so prefix it with an underscore since npm packages can't start with that
dir.join(format!("_{encoded_name}"))
} else {
// ensure backslashes are used on windows
for part in name.split('/') {
dir = dir.join(part);
}
dir
}
}
fn registry_folder(&self, registry_url: &Url) -> PathBuf {
self
.root_dir
.join(root_url_to_safe_local_dirname(registry_url))
}
pub fn resolve_package_folder_id_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Option<NpmPackageCacheFolderId> {
let mut maybe_relative_url = None;
// Iterate through known registries and try to get a match.
for registry_dirname in &self.known_registries_dirnames {
let registry_root_dir = self
.root_dir_url
.join(&format!("{}/", registry_dirname))
// this not succeeding indicates a fatal issue, so unwrap
.unwrap();
let Some(relative_url) = registry_root_dir.make_relative(specifier)
else {
continue;
};
if relative_url.starts_with("../") {
continue;
}
maybe_relative_url = Some(relative_url);
break;
}
let mut relative_url = maybe_relative_url?;
// base32 decode the url if it starts with an underscore
// * Ex. _{base32(package_name)}/
if let Some(end_url) = relative_url.strip_prefix('_') {
let mut parts = end_url
.split('/')
.map(ToOwned::to_owned)
.collect::<Vec<_>>();
match mixed_case_package_name_decode(&parts[0]) {
Some(part) => {
parts[0] = part;
}
None => return None,
}
relative_url = parts.join("/");
}
// examples:
// * chalk/5.0.1/
// * @types/chalk/5.0.1/
// * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps
let is_scoped_package = relative_url.starts_with('@');
let mut parts = relative_url
.split('/')
.enumerate()
.take(if is_scoped_package { 3 } else { 2 })
.map(|(_, part)| part)
.collect::<Vec<_>>();
if parts.len() < 2 {
return None;
}
let version_part = parts.pop().unwrap();
let name = parts.join("/");
let (version, copy_index) =
if let Some((version, copy_count)) = version_part.split_once('_') {
(version, copy_count.parse::<u8>().ok()?)
} else {
(version_part, 0)
};
Some(NpmPackageCacheFolderId {
nv: PackageNv {
name,
version: Version::parse_from_npm(version).ok()?,
},
copy_index,
})
}
pub fn get_cache_location(&self) -> PathBuf {
self.root_dir.clone()
}
}
pub fn mixed_case_package_name_encode(name: &str) -> String {
// use base32 encoding because it's reversible and the character set
// only includes the characters within 0-9 and A-Z so it can be lower cased
base32::encode(
base32::Alphabet::Rfc4648Lower { padding: false },
name.as_bytes(),
)
.to_lowercase()
}
pub fn mixed_case_package_name_decode(name: &str) -> Option<String> {
base32::decode(base32::Alphabet::Rfc4648Lower { padding: false }, name)
.and_then(|b| String::from_utf8(b).ok())
}
#[cfg(test)]
mod test {
use deno_core::url::Url;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use super::NpmCacheDir;
use crate::npm::cache_dir::NpmPackageCacheFolderId;
#[test]
fn should_get_package_folder() {
let deno_dir = crate::cache::DenoDir::new(None).unwrap();
let root_dir = deno_dir.npm_folder_path();
let registry_url = Url::parse("https://registry.npmjs.org/").unwrap();
let cache = NpmCacheDir::new(root_dir.clone(), vec![registry_url.clone()]);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "json".to_string(),
version: Version::parse_from_npm("1.2.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("json")
.join("1.2.5"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "json".to_string(),
version: Version::parse_from_npm("1.2.5").unwrap(),
},
copy_index: 1,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("json")
.join("1.2.5_1"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "JSON".to_string(),
version: Version::parse_from_npm("2.1.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("_jjju6tq")
.join("2.1.5"),
);
assert_eq!(
cache.package_folder_for_id(
&NpmPackageCacheFolderId {
nv: PackageNv {
name: "@types/JSON".to_string(),
version: Version::parse_from_npm("2.1.5").unwrap(),
},
copy_index: 0,
},
&registry_url,
),
root_dir
.join("registry.npmjs.org")
.join("_ib2hs4dfomxuuu2pjy")
.join("2.1.5"),
);
}
}

44
cli/npm/managed/cache/mod.rs vendored
View file

@ -8,6 +8,7 @@ use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
@ -18,10 +19,10 @@ use deno_npm::npm_rc::ResolvedNpmRc;
use deno_npm::registry::NpmPackageInfo;
use deno_npm::NpmPackageCacheFolderId;
use deno_semver::package::PackageNv;
use deno_semver::Version;
use crate::args::CacheSetting;
use crate::cache::CACHE_PERM;
use crate::npm::NpmCacheDir;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::hard_link_dir_recursive;
@ -87,9 +88,12 @@ impl NpmCache {
) -> Result<(), AnyError> {
let registry_url = self.npmrc.get_registry_url(&folder_id.nv.name);
assert_ne!(folder_id.copy_index, 0);
let package_folder = self
.cache_dir
.package_folder_for_id(folder_id, registry_url);
let package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
folder_id.copy_index,
registry_url,
);
if package_folder.exists()
// if this file exists, then the package didn't successfully initialize
@ -100,9 +104,12 @@ impl NpmCache {
return Ok(());
}
let original_package_folder = self
.cache_dir
.package_folder_for_nv(&folder_id.nv, registry_url);
let original_package_folder = self.cache_dir.package_folder_for_id(
&folder_id.nv.name,
&folder_id.nv.version.to_string(),
0, // original copy index
registry_url,
);
// it seems Windows does an "AccessDenied" error when moving a
// directory with hard links, so that's why this solution is done
@ -114,7 +121,12 @@ impl NpmCache {
pub fn package_folder_for_id(&self, id: &NpmPackageCacheFolderId) -> PathBuf {
let registry_url = self.npmrc.get_registry_url(&id.nv.name);
self.cache_dir.package_folder_for_id(id, registry_url)
self.cache_dir.package_folder_for_id(
&id.nv.name,
&id.nv.version.to_string(),
id.copy_index,
registry_url,
)
}
pub fn package_folder_for_nv(&self, package: &PackageNv) -> PathBuf {
@ -127,7 +139,12 @@ impl NpmCache {
package: &PackageNv,
registry_url: &Url,
) -> PathBuf {
self.cache_dir.package_folder_for_nv(package, registry_url)
self.cache_dir.package_folder_for_id(
&package.name,
&package.version.to_string(),
0, // original copy_index
registry_url,
)
}
pub fn package_name_folder(&self, name: &str) -> PathBuf {
@ -146,6 +163,15 @@ impl NpmCache {
self
.cache_dir
.resolve_package_folder_id_from_specifier(specifier)
.and_then(|cache_id| {
Some(NpmPackageCacheFolderId {
nv: PackageNv {
name: cache_id.name,
version: Version::parse_from_npm(&cache_id.version).ok()?,
},
copy_index: cache_id.copy_index,
})
})
}
pub fn load_package_info(

4
cli/npm/managed/mod.rs
View file

@ -7,6 +7,7 @@ use std::sync::Arc;
use cache::RegistryInfoDownloader;
use cache::TarballCache;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::NpmCacheDir;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::serde_json;
@ -35,6 +36,7 @@ use crate::args::LifecycleScriptsConfig;
use crate::args::NpmInstallDepsProvider;
use crate::args::NpmProcessState;
use crate::args::NpmProcessStateKind;
use crate::cache::DenoCacheEnvFsAdapter;
use crate::cache::FastInsecureHasher;
use crate::http_util::HttpClientProvider;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
@ -50,7 +52,6 @@ use self::resolvers::NpmPackageFsResolver;
use super::CliNpmResolver;
use super::InnerCliNpmResolverRef;
use super::NpmCacheDir;
mod cache;
mod registry;
@ -188,6 +189,7 @@ fn create_inner(
fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc<NpmCache> {
Arc::new(NpmCache::new(
NpmCacheDir::new(
&DenoCacheEnvFsAdapter(options.fs.as_ref()),
options.npm_global_cache_dir.clone(),
options.npmrc.get_all_known_registries_urls(),
),

4
cli/npm/managed/resolvers/local.rs
View file

@ -19,6 +19,8 @@ use crate::args::LifecycleScriptsConfig;
use crate::colors;
use async_trait::async_trait;
use deno_ast::ModuleSpecifier;
use deno_cache_dir::npm::mixed_case_package_name_decode;
use deno_cache_dir::npm::mixed_case_package_name_encode;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures::stream::FuturesUnordered;
@ -42,8 +44,6 @@ use serde::Serialize;
use crate::args::NpmInstallDepsProvider;
use crate::cache::CACHE_PERM;
use crate::npm::cache_dir::mixed_case_package_name_decode;
use crate::npm::cache_dir::mixed_case_package_name_encode;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
use crate::util::fs::clone_dir_recursive;

2
cli/npm/mod.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod byonm;
mod cache_dir;
mod common;
mod managed;
@ -24,7 +23,6 @@ use crate::file_fetcher::FileFetcher;
pub use self::byonm::ByonmCliNpmResolver;
pub use self::byonm::CliNpmResolverByonmCreateOptions;
pub use self::cache_dir::NpmCacheDir;
pub use self::managed::CliNpmResolverManagedCreateOptions;
pub use self::managed::CliNpmResolverManagedSnapshotOption;
pub use self::managed::ManagedCliNpmResolver;

4
cli/standalone/file_system.rs
View file

@ -102,7 +102,7 @@ impl FileSystem for DenoCompileFileSystem {
&self,
path: &Path,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(path)?;
RealFs.mkdir_sync(path, recursive, mode)
@ -111,7 +111,7 @@ impl FileSystem for DenoCompileFileSystem {
&self,
path: PathBuf,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()> {
self.error_if_in_vfs(&path)?;
RealFs.mkdir_async(path, recursive, mode).await

4
cli/standalone/mod.rs
View file

@ -6,6 +6,7 @@
#![allow(unused_imports)]
use deno_ast::MediaType;
use deno_cache_dir::npm::NpmCacheDir;
use deno_config::workspace::MappedResolution;
use deno_config::workspace::MappedResolutionError;
use deno_config::workspace::ResolverWorkspaceJsrPackage;
@ -55,6 +56,7 @@ use crate::args::StorageKeyResolver;
use crate::cache::Caches;
use crate::cache::DenoDirProvider;
use crate::cache::NodeAnalysisCache;
use crate::cache::RealDenoCacheEnv;
use crate::http_util::HttpClientProvider;
use crate::node::CliCjsCodeAnalyzer;
use crate::npm::create_cli_npm_resolver;
@ -62,7 +64,6 @@ use crate::npm::CliNpmResolverByonmCreateOptions;
use crate::npm::CliNpmResolverCreateOptions;
use crate::npm::CliNpmResolverManagedCreateOptions;
use crate::npm::CliNpmResolverManagedSnapshotOption;
use crate::npm::NpmCacheDir;
use crate::resolver::CjsResolutionStore;
use crate::resolver::CliNodeResolver;
use crate::resolver::NpmModuleLoader;
@ -464,6 +465,7 @@ pub async fn run(
let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();
let root_node_modules_path = root_path.join("node_modules");
let npm_cache_dir = NpmCacheDir::new(
&RealDenoCacheEnv,
root_node_modules_path.clone(),
vec![npm_registry_url.clone()],
);

157
cli/util/fs.rs
View file

@ -37,10 +37,98 @@ pub fn atomic_write_file_with_retries<T: AsRef<[u8]>>(
file_path: &Path,
data: T,
mode: u32,
) -> std::io::Result<()> {
struct RealAtomicWriteFileFs {
mode: u32,
}
impl AtomicWriteFileFs for RealAtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
write_file(path, bytes, self.mode)
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
std::fs::rename(from, to)
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
std::fs::remove_file(path)
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
std::fs::create_dir_all(dir_path)
}
fn path_exists(&self, path: &Path) -> bool {
path.exists()
}
}
atomic_write_file_with_retries_and_fs(
&RealAtomicWriteFileFs { mode },
file_path,
data.as_ref(),
)
}
pub trait AtomicWriteFileFs {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()>;
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()>;
fn remove_file(&self, path: &Path) -> std::io::Result<()>;
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()>;
fn path_exists(&self, path: &Path) -> bool;
}
pub struct AtomicWriteFileFsAdapter<'a> {
pub fs: &'a dyn FileSystem,
pub write_mode: u32,
}
impl<'a> AtomicWriteFileFs for AtomicWriteFileFsAdapter<'a> {
fn write_file(&self, path: &Path, bytes: &[u8]) -> std::io::Result<()> {
self
.fs
.write_file_sync(
path,
deno_runtime::deno_fs::OpenOptions::write(
true,
false,
false,
Some(self.write_mode),
),
None,
bytes,
)
.map_err(|e| e.into_io_error())
}
fn rename_file(&self, from: &Path, to: &Path) -> std::io::Result<()> {
self.fs.rename_sync(from, to).map_err(|e| e.into_io_error())
}
fn remove_file(&self, path: &Path) -> std::io::Result<()> {
self
.fs
.remove_sync(path, false)
.map_err(|e| e.into_io_error())
}
fn create_dir_all(&self, dir_path: &Path) -> std::io::Result<()> {
self
.fs
.mkdir_sync(dir_path, /* recursive */ true, None)
.map_err(|e| e.into_io_error())
}
fn path_exists(&self, path: &Path) -> bool {
self.fs.exists_sync(path)
}
}
pub fn atomic_write_file_with_retries_and_fs<T: AsRef<[u8]>>(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: T,
) -> std::io::Result<()> {
let mut count = 0;
loop {
match atomic_write_file(file_path, data.as_ref(), mode) {
match atomic_write_file(fs, file_path, data.as_ref()) {
Ok(()) => return Ok(()),
Err(err) => {
if count >= 5 {
@ -61,63 +149,54 @@ pub fn atomic_write_file_with_retries<T: AsRef<[u8]>>(
///
/// This also handles creating the directory if a NotFound error
/// occurs.
fn atomic_write_file<T: AsRef<[u8]>>(
fn atomic_write_file(
fs: &impl AtomicWriteFileFs,
file_path: &Path,
data: T,
mode: u32,
data: &[u8],
) -> std::io::Result<()> {
fn atomic_write_file_raw(
fs: &impl AtomicWriteFileFs,
temp_file_path: &Path,
file_path: &Path,
data: &[u8],
mode: u32,
) -> std::io::Result<()> {
write_file(temp_file_path, data, mode)?;
std::fs::rename(temp_file_path, file_path).map_err(|err| {
fs.write_file(temp_file_path, data)?;
fs.rename_file(temp_file_path, file_path).map_err(|err| {
// clean up the created temp file on error
let _ = std::fs::remove_file(temp_file_path);
let _ = fs.remove_file(temp_file_path);
err
})
}
fn inner(file_path: &Path, data: &[u8], mode: u32) -> std::io::Result<()> {
let temp_file_path = get_atomic_file_path(file_path);
let temp_file_path = get_atomic_file_path(file_path);
if let Err(write_err) =
atomic_write_file_raw(&temp_file_path, file_path, data, mode)
{
if write_err.kind() == ErrorKind::NotFound {
let parent_dir_path = file_path.parent().unwrap();
match std::fs::create_dir_all(parent_dir_path) {
Ok(()) => {
return atomic_write_file_raw(
&temp_file_path,
file_path,
data,
mode,
)
if let Err(write_err) =
atomic_write_file_raw(fs, &temp_file_path, file_path, data)
{
if write_err.kind() == ErrorKind::NotFound {
let parent_dir_path = file_path.parent().unwrap();
match fs.create_dir_all(parent_dir_path) {
Ok(()) => {
return atomic_write_file_raw(fs, &temp_file_path, file_path, data)
.map_err(|err| add_file_context_to_err(file_path, err));
}
Err(create_err) => {
if !parent_dir_path.exists() {
return Err(Error::new(
create_err.kind(),
format!(
"{:#} (for '{}')\nCheck the permission of the directory.",
create_err,
parent_dir_path.display()
),
));
}
}
Err(create_err) => {
if !fs.path_exists(parent_dir_path) {
return Err(Error::new(
create_err.kind(),
format!(
"{:#} (for '{}')\nCheck the permission of the directory.",
create_err,
parent_dir_path.display()
),
));
}
}
}
return Err(add_file_context_to_err(file_path, write_err));
}
Ok(())
return Err(add_file_context_to_err(file_path, write_err));
}
inner(file_path, data.as_ref(), mode)
Ok(())
}
/// Creates a std::fs::File handling if the parent does not exist.

42
cli/util/path.rs
View file

@ -165,48 +165,6 @@ pub fn relative_path(from: &Path, to: &Path) -> Option<PathBuf> {
pathdiff::diff_paths(to, from)
}
/// Gets if the provided character is not supported on all
/// kinds of file systems.
pub fn is_banned_path_char(c: char) -> bool {
matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*')
}
/// Gets a safe local directory name for the provided url.
///
/// For example:
/// https://deno.land:8080/path -> deno.land_8080/path
pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf {
fn sanitize_segment(text: &str) -> String {
text
.chars()
.map(|c| if is_banned_segment_char(c) { '_' } else { c })
.collect()
}
fn is_banned_segment_char(c: char) -> bool {
matches!(c, '/' | '\\') || is_banned_path_char(c)
}
let mut result = String::new();
if let Some(domain) = root.domain() {
result.push_str(&sanitize_segment(domain));
}
if let Some(port) = root.port() {
if !result.is_empty() {
result.push('_');
}
result.push_str(&port.to_string());
}
let mut result = PathBuf::from(result);
if let Some(segments) = root.path_segments() {
for segment in segments.filter(|s| !s.is_empty()) {
result = result.join(sanitize_segment(segment));
}
}
result
}
/// Slightly different behaviour than the default matching
/// where an exact path needs to be matched to be opted-in
/// rather than just a partial directory match.

8
ext/fs/in_memory_fs.rs
View file

@ -44,7 +44,7 @@ impl InMemoryFs {
pub fn setup_text_files(&self, files: Vec<(String, String)>) {
for (path, text) in files {
let path = PathBuf::from(path);
self.mkdir_sync(path.parent().unwrap(), true, 0).unwrap();
self.mkdir_sync(path.parent().unwrap(), true, None).unwrap();
self
.write_file_sync(
&path,
@ -101,7 +101,7 @@ impl FileSystem for InMemoryFs {
&self,
path: &Path,
recursive: bool,
_mode: u32,
_mode: Option<u32>,
) -> FsResult<()> {
let path = normalize_path(path);
@ -119,7 +119,7 @@ impl FileSystem for InMemoryFs {
},
None => {
if recursive {
self.mkdir_sync(parent, true, 0)?;
self.mkdir_sync(parent, true, None)?;
} else {
return Err(FsError::Io(Error::new(
ErrorKind::NotFound,
@ -149,7 +149,7 @@ impl FileSystem for InMemoryFs {
&self,
path: PathBuf,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()> {
self.mkdir_sync(&path, recursive, mode)
}

10
ext/fs/interface.rs
View file

@ -121,13 +121,17 @@ pub trait FileSystem: std::fmt::Debug + MaybeSend + MaybeSync {
access_check: Option<AccessCheckCb<'a>>,
) -> FsResult<Rc<dyn File>>;
fn mkdir_sync(&self, path: &Path, recursive: bool, mode: u32)
-> FsResult<()>;
fn mkdir_sync(
&self,
path: &Path,
recursive: bool,
mode: Option<u32>,
) -> FsResult<()>;
async fn mkdir_async(
&self,
path: PathBuf,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()>;
fn chmod_sync(&self, path: &Path, mode: u32) -> FsResult<()>;

12
ext/fs/ops.rs
View file

@ -197,7 +197,7 @@ where
.check_write(&path, "Deno.mkdirSync()")?;
let fs = state.borrow::<FileSystemRc>();
fs.mkdir_sync(&path, recursive, mode)
fs.mkdir_sync(&path, recursive, Some(mode))
.context_path("mkdir", &path)?;
Ok(())
@ -221,7 +221,7 @@ where
(state.borrow::<FileSystemRc>().clone(), path)
};
fs.mkdir_async(path.clone(), recursive, mode)
fs.mkdir_async(path.clone(), recursive, Some(mode))
.await
.context_path("mkdir", &path)?;
@ -886,7 +886,7 @@ where
const MAX_TRIES: u32 = 10;
for _ in 0..MAX_TRIES {
let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?;
match fs.mkdir_sync(&path, false, 0o700) {
match fs.mkdir_sync(&path, false, Some(0o700)) {
Ok(_) => {
// PERMISSIONS: ensure the absolute path is not leaked
let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?;
@ -928,7 +928,11 @@ where
const MAX_TRIES: u32 = 10;
for _ in 0..MAX_TRIES {
let path = tmp_name(&mut rng, &dir, prefix.as_deref(), suffix.as_deref())?;
match fs.clone().mkdir_async(path.clone(), false, 0o700).await {
match fs
.clone()
.mkdir_async(path.clone(), false, Some(0o700))
.await
{
Ok(_) => {
// PERMISSIONS: ensure the absolute path is not leaked
let path = strip_dir_prefix(&dir, dir_arg.as_deref(), path)?;

8
ext/fs/std_fs.rs
View file

@ -101,7 +101,7 @@ impl FileSystem for RealFs {
&self,
path: &Path,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()> {
mkdir(path, recursive, mode)
}
@ -109,7 +109,7 @@ impl FileSystem for RealFs {
&self,
path: PathBuf,
recursive: bool,
mode: u32,
mode: Option<u32>,
) -> FsResult<()> {
spawn_blocking(move || mkdir(&path, recursive, mode)).await?
}
@ -407,11 +407,11 @@ impl FileSystem for RealFs {
}
}
fn mkdir(path: &Path, recursive: bool, mode: u32) -> FsResult<()> {
fn mkdir(path: &Path, recursive: bool, mode: Option<u32>) -> FsResult<()> {
let mut builder = fs::DirBuilder::new();
builder.recursive(recursive);
#[cfg(unix)]
{
if let Some(mode) = mode {
use std::os::unix::fs::DirBuilderExt;
builder.mode(mode);
}