From 2d4c46c975eb916dc622cc729a1a8d397582a76f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 28 Nov 2022 17:28:54 -0500 Subject: [PATCH] refactor: create util folder, move nap_sym to napi/sym, move http_cache to cache folder (#16857) --- Cargo.toml | 4 +- cli/args/config_file.rs | 9 +- cli/args/lockfile.rs | 7 +- cli/args/mod.rs | 126 +++- cli/cache/disk_cache.rs | 7 +- cli/{ => cache}/http_cache.rs | 27 +- cli/cache/mod.rs | 6 + cli/file_fetcher.rs | 219 ++----- cli/lsp/cache.rs | 12 +- cli/lsp/completions.rs | 8 +- cli/lsp/config.rs | 7 +- cli/lsp/documents.rs | 13 +- cli/lsp/language_server.rs | 41 +- cli/lsp/registries.rs | 4 +- cli/lsp/testing/definitions.rs | 2 +- cli/lsp/testing/execution.rs | 2 +- cli/lsp/tsc.rs | 6 +- cli/lsp/urls.rs | 2 +- cli/main.rs | 33 +- cli/module_loader.rs | 4 +- cli/{napi_sym => napi/sym}/Cargo.toml | 0 cli/{napi_sym => napi/sym}/README.md | 0 cli/{napi_sym => napi/sym}/lib.rs | 2 +- .../sym}/symbol_exports.json | 0 cli/npm/cache.rs | 21 +- cli/npm/registry.rs | 10 +- cli/npm/resolvers/global.rs | 3 +- cli/npm/resolvers/local.rs | 15 +- cli/npm/resolvers/mod.rs | 4 +- cli/proc_state.rs | 6 +- cli/tools/bench.rs | 22 +- cli/tools/coverage/mod.rs | 4 +- cli/tools/fmt.rs | 14 +- cli/tools/info.rs | 2 +- cli/tools/installer.rs | 13 +- cli/tools/lint.rs | 10 +- cli/tools/standalone.rs | 4 +- cli/tools/task.rs | 4 +- cli/tools/test.rs | 117 +++- cli/tools/vendor/mappings.rs | 4 +- cli/tools/vendor/mod.rs | 20 +- cli/tools/vendor/specifiers.rs | 15 +- cli/tsc/mod.rs | 7 +- cli/{ => util}/checksum.rs | 0 cli/{ => util}/diff.rs | 0 cli/{ => util}/display.rs | 0 cli/{ => util}/file_watcher.rs | 2 +- cli/{fs_util.rs => util/fs.rs} | 552 +----------------- cli/{ => util}/logger.rs | 0 cli/util/mod.rs | 14 + cli/util/path.rs | 452 ++++++++++++++ cli/{ => util}/progress_bar.rs | 0 cli/{ => util}/text_encoding.rs | 0 cli/{unix_util.rs => util/unix.rs} | 0 cli/{windows_util.rs => util/windows.rs} | 0 cli/worker.rs | 2 +- tools/napi/generate_symbols_lists.js | 2 +- 57 files changed, 940 insertions(+), 920 deletions(-) rename cli/{ => cache}/http_cache.rs (92%) rename cli/{napi_sym => napi/sym}/Cargo.toml (100%) rename cli/{napi_sym => napi/sym}/README.md (100%) rename cli/{napi_sym => napi/sym}/lib.rs (95%) rename cli/{napi_sym => napi/sym}/symbol_exports.json (100%) rename cli/{ => util}/checksum.rs (100%) rename cli/{ => util}/diff.rs (100%) rename cli/{ => util}/display.rs (100%) rename cli/{ => util}/file_watcher.rs (99%) rename cli/{fs_util.rs => util/fs.rs} (52%) rename cli/{ => util}/logger.rs (100%) create mode 100644 cli/util/mod.rs create mode 100644 cli/util/path.rs rename cli/{ => util}/progress_bar.rs (100%) rename cli/{ => util}/text_encoding.rs (100%) rename cli/{unix_util.rs => util/unix.rs} (100%) rename cli/{windows_util.rs => util/windows.rs} (100%) diff --git a/Cargo.toml b/Cargo.toml index c797d1c870..6c70909464 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" members = [ "bench_util", "cli", - "cli/napi_sym", + "cli/napi/sym", "core", "ops", "runtime", @@ -47,7 +47,7 @@ deno_core = { version = "0.161.0", path = "./core" } deno_ops = { version = "0.39.0", path = "./ops" } serde_v8 = { version = "0.72.0", path = "./serde_v8" } deno_runtime = { version = "0.87.0", path = "./runtime" } -napi_sym = { version = "0.9.0", path = "./cli/napi_sym" } +napi_sym = { version = "0.9.0", path = "./cli/napi/sym" } deno_bench_util = { version = "0.73.0", path = "./bench_util" } test_util = { path = "./test_util" } diff --git a/cli/args/config_file.rs b/cli/args/config_file.rs index 435e0d715b..76340aa8ba 100644 --- a/cli/args/config_file.rs +++ b/cli/args/config_file.rs @@ -3,10 +3,9 @@ use crate::args::ConfigFlag; use crate::args::Flags; use crate::args::TaskFlags; -use crate::fs_util; -use crate::fs_util::canonicalize_path; -use crate::fs_util::specifier_parent; -use crate::fs_util::specifier_to_file_path; +use crate::util::fs::canonicalize_path; +use crate::util::path::specifier_parent; +use crate::util::path::specifier_to_file_path; use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; @@ -467,7 +466,7 @@ impl ConfigFile { .. }) = &flags.subcommand { - let task_cwd = fs_util::canonicalize_path(&PathBuf::from(path))?; + let task_cwd = canonicalize_path(&PathBuf::from(path))?; if let Some(path) = Self::discover_from(&task_cwd, &mut checked)? { return Ok(Some(path)); } diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index f99d2f5701..73a075f814 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -20,6 +20,7 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; use crate::tools::fmt::format_json; +use crate::util; use crate::Flags; #[derive(Debug)] @@ -260,7 +261,7 @@ impl Lockfile { /// is not included, insert it. fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { if let Some(lockfile_checksum) = self.content.remote.get(specifier) { - let compiled_checksum = crate::checksum::gen(&[code.as_bytes()]); + let compiled_checksum = util::checksum::gen(&[code.as_bytes()]); lockfile_checksum == &compiled_checksum } else { self.insert(specifier, code); @@ -269,7 +270,7 @@ impl Lockfile { } fn insert(&mut self, specifier: &str, code: &str) { - let checksum = crate::checksum::gen(&[code.as_bytes()]); + let checksum = util::checksum::gen(&[code.as_bytes()]); self.content.remote.insert(specifier.to_string(), checksum); self.has_content_changed = true; } @@ -359,7 +360,7 @@ impl deno_graph::source::Locker for Locker { } fn get_checksum(&self, content: &str) -> String { - crate::checksum::gen(&[content.as_bytes()]) + util::checksum::gen(&[content.as_bytes()]) } fn get_filename(&self) -> Option { diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 50a407ee3d..d1ff39f98e 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -22,6 +22,10 @@ pub use config_file::TsConfig; pub use config_file::TsConfigForEmit; pub use config_file::TsConfigType; pub use config_file::TsTypeLib; +use deno_runtime::deno_tls::rustls; +use deno_runtime::deno_tls::rustls_native_certs::load_native_certs; +use deno_runtime::deno_tls::rustls_pemfile; +use deno_runtime::deno_tls::webpki_roots; pub use flags::*; pub use lockfile::Lockfile; pub use lockfile::LockfileError; @@ -40,16 +44,130 @@ use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::PermissionsOptions; use std::collections::BTreeMap; use std::env; +use std::io::BufReader; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; use crate::cache::DenoDir; -use crate::file_fetcher::get_root_cert_store; -use crate::file_fetcher::CacheSetting; -use crate::fs_util; +use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::version; +/// Indicates how cached source files should be handled. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum CacheSetting { + /// Only the cached files should be used. Any files not in the cache will + /// error. This is the equivalent of `--cached-only` in the CLI. + Only, + /// No cached source files should be used, and all files should be reloaded. + /// This is the equivalent of `--reload` in the CLI. + ReloadAll, + /// Only some cached resources should be used. This is the equivalent of + /// `--reload=https://deno.land/std` or + /// `--reload=https://deno.land/std,https://deno.land/x/example`. + ReloadSome(Vec), + /// The usability of a cached value is determined by analyzing the cached + /// headers and other metadata associated with a cached response, reloading + /// any cached "non-fresh" cached responses. + RespectHeaders, + /// The cached source files should be used for local modules. This is the + /// default behavior of the CLI. + Use, +} + +impl CacheSetting { + pub fn should_use_for_npm_package(&self, package_name: &str) -> bool { + match self { + CacheSetting::ReloadAll => false, + CacheSetting::ReloadSome(list) => { + if list.iter().any(|i| i == "npm:") { + return false; + } + let specifier = format!("npm:{}", package_name); + if list.contains(&specifier) { + return false; + } + true + } + _ => true, + } + } +} + +/// Create and populate a root cert store based on the passed options and +/// environment. +pub fn get_root_cert_store( + maybe_root_path: Option, + maybe_ca_stores: Option>, + maybe_ca_file: Option, +) -> Result { + let mut root_cert_store = RootCertStore::empty(); + let ca_stores: Vec = maybe_ca_stores + .or_else(|| { + let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?; + Some( + env_ca_store + .split(',') + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()) + .collect(), + ) + }) + .unwrap_or_else(|| vec!["mozilla".to_string()]); + + for store in ca_stores.iter() { + match store.as_str() { + "mozilla" => { + root_cert_store.add_server_trust_anchors( + webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| { + rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( + ta.subject, + ta.spki, + ta.name_constraints, + ) + }), + ); + } + "system" => { + let roots = load_native_certs().expect("could not load platform certs"); + for root in roots { + root_cert_store + .add(&rustls::Certificate(root.0)) + .expect("Failed to add platform cert to root cert store"); + } + } + _ => { + return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store)); + } + } + } + + let ca_file = maybe_ca_file.or_else(|| env::var("DENO_CERT").ok()); + if let Some(ca_file) = ca_file { + let ca_file = if let Some(root) = &maybe_root_path { + root.join(&ca_file) + } else { + PathBuf::from(ca_file) + }; + let certfile = std::fs::File::open(&ca_file)?; + let mut reader = BufReader::new(certfile); + + match rustls_pemfile::certs(&mut reader) { + Ok(certs) => { + root_cert_store.add_parsable_certificates(&certs); + } + Err(e) => { + return Err(anyhow!( + "Unable to add pem file to certificate store: {}", + e + )); + } + } + } + + Ok(root_cert_store) +} + /// Overrides for the options below that when set will /// use these values over the values derived from the /// CLI flags or config file. @@ -176,7 +294,7 @@ impl CliOptions { } else { std::env::current_dir()?.join("node_modules") }; - Ok(Some(fs_util::canonicalize_path_maybe_not_exists(&path)?)) + Ok(Some(canonicalize_path_maybe_not_exists(&path)?)) } pub fn resolve_root_cert_store(&self) -> Result { diff --git a/cli/cache/disk_cache.rs b/cli/cache/disk_cache.rs index 81379ac94a..60e353d852 100644 --- a/cli/cache/disk_cache.rs +++ b/cli/cache/disk_cache.rs @@ -1,7 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use crate::fs_util; -use crate::http_cache::url_to_filename; +use super::http_cache::url_to_filename; +use super::CACHE_PERM; +use crate::util::fs::atomic_write_file; use deno_core::url::Host; use deno_core::url::Url; @@ -144,7 +145,7 @@ impl DiskCache { Some(parent) => self.ensure_dir_exists(parent), None => Ok(()), }?; - fs_util::atomic_write_file(&path, data, crate::http_cache::CACHE_PERM) + atomic_write_file(&path, data, CACHE_PERM) .map_err(|e| with_io_context(&e, format!("{:#?}", &path))) } } diff --git a/cli/http_cache.rs b/cli/cache/http_cache.rs similarity index 92% rename from cli/http_cache.rs rename to cli/cache/http_cache.rs index 6cda9f2794..f4cf3ef110 100644 --- a/cli/http_cache.rs +++ b/cli/cache/http_cache.rs @@ -3,8 +3,8 @@ //! as defined in RFC 7234 (). //! Currently it's a very simplified version to fulfill Deno needs //! at hand. -use crate::fs_util; use crate::http_util::HeadersMap; +use crate::util; use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::serde::Deserialize; @@ -19,7 +19,7 @@ use std::path::Path; use std::path::PathBuf; use std::time::SystemTime; -pub const CACHE_PERM: u32 = 0o644; +use super::CACHE_PERM; /// Turn base of url (scheme, hostname, port) into a valid filename. /// This method replaces port part with a special string token (because @@ -68,31 +68,32 @@ pub fn url_to_filename(url: &Url) -> Option { // NOTE: fragment is omitted on purpose - it's not taken into // account when caching - it denotes parts of webpage, which // in case of static resources doesn't make much sense - let hashed_filename = crate::checksum::gen(&[rest_str.as_bytes()]); + let hashed_filename = util::checksum::gen(&[rest_str.as_bytes()]); cache_filename.push(hashed_filename); Some(cache_filename) } +/// Cached metadata about a url. #[derive(Serialize, Deserialize)] -pub struct Metadata { +pub struct CachedUrlMetadata { pub headers: HeadersMap, pub url: String, #[serde(default = "SystemTime::now")] pub now: SystemTime, } -impl Metadata { +impl CachedUrlMetadata { pub fn write(&self, cache_filename: &Path) -> Result<(), AnyError> { let metadata_filename = Self::filename(cache_filename); let json = serde_json::to_string_pretty(self)?; - fs_util::atomic_write_file(&metadata_filename, json, CACHE_PERM)?; + util::fs::atomic_write_file(&metadata_filename, json, CACHE_PERM)?; Ok(()) } - pub fn read(cache_filename: &Path) -> Result { - let metadata_filename = Metadata::filename(cache_filename); + pub fn read(cache_filename: &Path) -> Result { + let metadata_filename = Self::filename(cache_filename); let metadata = fs::read_to_string(metadata_filename)?; - let metadata: Metadata = serde_json::from_str(&metadata)?; + let metadata: Self = serde_json::from_str(&metadata)?; Ok(metadata) } @@ -149,10 +150,10 @@ impl HttpCache { url_to_filename(url) .ok_or_else(|| generic_error("Can't convert url to filename."))?, ); - let metadata_filename = Metadata::filename(&cache_filename); + let metadata_filename = CachedUrlMetadata::filename(&cache_filename); let file = File::open(cache_filename)?; let metadata = fs::read_to_string(metadata_filename)?; - let metadata: Metadata = serde_json::from_str(&metadata)?; + let metadata: CachedUrlMetadata = serde_json::from_str(&metadata)?; Ok((file, metadata.headers, metadata.now)) } @@ -172,9 +173,9 @@ impl HttpCache { .expect("Cache filename should have a parent dir"); self.ensure_dir_exists(parent_filename)?; // Cache content - fs_util::atomic_write_file(&cache_filename, content, CACHE_PERM)?; + util::fs::atomic_write_file(&cache_filename, content, CACHE_PERM)?; - let metadata = Metadata { + let metadata = CachedUrlMetadata { now: SystemTime::now(), url: url.to_string(), headers: headers_map, diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index cf9a4c4413..b0d79400aa 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -19,6 +19,7 @@ mod common; mod deno_dir; mod disk_cache; mod emit; +mod http_cache; mod incremental; mod node; mod parsed_source; @@ -28,10 +29,15 @@ pub use common::FastInsecureHasher; pub use deno_dir::DenoDir; pub use disk_cache::DiskCache; pub use emit::EmitCache; +pub use http_cache::CachedUrlMetadata; +pub use http_cache::HttpCache; pub use incremental::IncrementalCache; pub use node::NodeAnalysisCache; pub use parsed_source::ParsedSourceCache; +/// Permissions used to save a file in the disk caches. +pub const CACHE_PERM: u32 = 0o644; + /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index b9a91d41a0..12f39c7e30 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -1,18 +1,18 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use crate::args::CacheSetting; use crate::auth_tokens::AuthTokens; +use crate::cache::HttpCache; use crate::colors; -use crate::http_cache::HttpCache; use crate::http_util::CacheSemantics; use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceResult; use crate::http_util::HttpClient; -use crate::progress_bar::ProgressBar; -use crate::text_encoding; +use crate::util::progress_bar::ProgressBar; +use crate::util::text_encoding; use data_url::DataUrl; use deno_ast::MediaType; -use deno_core::anyhow::anyhow; use deno_core::error::custom_error; use deno_core::error::generic_error; use deno_core::error::uri_error; @@ -21,11 +21,6 @@ use deno_core::futures; use deno_core::futures::future::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; -use deno_runtime::deno_tls::rustls; -use deno_runtime::deno_tls::rustls::RootCertStore; -use deno_runtime::deno_tls::rustls_native_certs::load_native_certs; -use deno_runtime::deno_tls::rustls_pemfile; -use deno_runtime::deno_tls::webpki_roots; use deno_runtime::deno_web::BlobStore; use deno_runtime::permissions::Permissions; use log::debug; @@ -34,7 +29,6 @@ use std::collections::HashMap; use std::env; use std::fs; use std::future::Future; -use std::io::BufReader; use std::io::Read; use std::path::PathBuf; use std::pin::Pin; @@ -82,86 +76,6 @@ impl FileCache { } } -/// Indicates how cached source files should be handled. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum CacheSetting { - /// Only the cached files should be used. Any files not in the cache will - /// error. This is the equivalent of `--cached-only` in the CLI. - Only, - /// No cached source files should be used, and all files should be reloaded. - /// This is the equivalent of `--reload` in the CLI. - ReloadAll, - /// Only some cached resources should be used. This is the equivalent of - /// `--reload=https://deno.land/std` or - /// `--reload=https://deno.land/std,https://deno.land/x/example`. - ReloadSome(Vec), - /// The usability of a cached value is determined by analyzing the cached - /// headers and other metadata associated with a cached response, reloading - /// any cached "non-fresh" cached responses. - RespectHeaders, - /// The cached source files should be used for local modules. This is the - /// default behavior of the CLI. - Use, -} - -impl CacheSetting { - /// Returns if the cache should be used for a given specifier. - pub fn should_use( - &self, - specifier: &ModuleSpecifier, - http_cache: &HttpCache, - ) -> bool { - match self { - CacheSetting::ReloadAll => false, - CacheSetting::Use | CacheSetting::Only => true, - CacheSetting::RespectHeaders => { - if let Ok((_, headers, cache_time)) = http_cache.get(specifier) { - let cache_semantics = - CacheSemantics::new(headers, cache_time, SystemTime::now()); - cache_semantics.should_use() - } else { - false - } - } - CacheSetting::ReloadSome(list) => { - let mut url = specifier.clone(); - url.set_fragment(None); - if list.contains(&url.as_str().to_string()) { - return false; - } - url.set_query(None); - let mut path = PathBuf::from(url.as_str()); - loop { - if list.contains(&path.to_str().unwrap().to_string()) { - return false; - } - if !path.pop() { - break; - } - } - true - } - } - } - - pub fn should_use_for_npm_package(&self, package_name: &str) -> bool { - match self { - CacheSetting::ReloadAll => false, - CacheSetting::ReloadSome(list) => { - if list.contains(&"npm:".to_string()) { - return false; - } - let specifier = format!("npm:{}", package_name); - if list.contains(&specifier) { - return false; - } - true - } - _ => true, - } - } -} - /// Fetch a source file from the local file system. fn fetch_local(specifier: &ModuleSpecifier) -> Result { let local = specifier.to_file_path().map_err(|_| { @@ -182,80 +96,6 @@ fn fetch_local(specifier: &ModuleSpecifier) -> Result { }) } -/// Create and populate a root cert store based on the passed options and -/// environment. -pub fn get_root_cert_store( - maybe_root_path: Option, - maybe_ca_stores: Option>, - maybe_ca_file: Option, -) -> Result { - let mut root_cert_store = RootCertStore::empty(); - let ca_stores: Vec = maybe_ca_stores - .or_else(|| { - let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?; - Some( - env_ca_store - .split(',') - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .collect(), - ) - }) - .unwrap_or_else(|| vec!["mozilla".to_string()]); - - for store in ca_stores.iter() { - match store.as_str() { - "mozilla" => { - root_cert_store.add_server_trust_anchors( - webpki_roots::TLS_SERVER_ROOTS.0.iter().map(|ta| { - rustls::OwnedTrustAnchor::from_subject_spki_name_constraints( - ta.subject, - ta.spki, - ta.name_constraints, - ) - }), - ); - } - "system" => { - let roots = load_native_certs().expect("could not load platform certs"); - for root in roots { - root_cert_store - .add(&rustls::Certificate(root.0)) - .expect("Failed to add platform cert to root cert store"); - } - } - _ => { - return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store)); - } - } - } - - let ca_file = maybe_ca_file.or_else(|| env::var("DENO_CERT").ok()); - if let Some(ca_file) = ca_file { - let ca_file = if let Some(root) = &maybe_root_path { - root.join(&ca_file) - } else { - PathBuf::from(ca_file) - }; - let certfile = fs::File::open(&ca_file)?; - let mut reader = BufReader::new(certfile); - - match rustls_pemfile::certs(&mut reader) { - Ok(certs) => { - root_cert_store.add_parsable_certificates(&certs); - } - Err(e) => { - return Err(anyhow!( - "Unable to add pem file to certificate store: {}", - e - )); - } - } - } - - Ok(root_cert_store) -} - /// Returns the decoded body and content-type of a provided /// data URL. pub fn get_source_from_data_url( @@ -571,7 +411,7 @@ impl FileFetcher { return futures::future::err(err).boxed(); } - if self.cache_setting.should_use(specifier, &self.http_cache) { + if self.should_use_cache(specifier) { match self.fetch_cached(specifier, redirect_limit) { Ok(Some(file)) => { return futures::future::ok(file).boxed(); @@ -654,6 +494,41 @@ impl FileFetcher { .boxed() } + /// Returns if the cache should be used for a given specifier. + fn should_use_cache(&self, specifier: &ModuleSpecifier) -> bool { + match &self.cache_setting { + CacheSetting::ReloadAll => false, + CacheSetting::Use | CacheSetting::Only => true, + CacheSetting::RespectHeaders => { + if let Ok((_, headers, cache_time)) = self.http_cache.get(specifier) { + let cache_semantics = + CacheSemantics::new(headers, cache_time, SystemTime::now()); + cache_semantics.should_use() + } else { + false + } + } + CacheSetting::ReloadSome(list) => { + let mut url = specifier.clone(); + url.set_fragment(None); + if list.contains(&url.as_str().to_string()) { + return false; + } + url.set_query(None); + let mut path = PathBuf::from(url.as_str()); + loop { + if list.contains(&path.to_str().unwrap().to_string()) { + return false; + } + if !path.pop() { + break; + } + } + true + } + } + } + /// Fetch a source file and asynchronously return it. pub async fn fetch( &self, @@ -754,6 +629,7 @@ impl FileFetcher { #[cfg(test)] mod tests { + use crate::cache::CachedUrlMetadata; use crate::http_util::HttpClient; use super::*; @@ -1175,8 +1051,7 @@ mod tests { .http_cache .get_cache_filename(&specifier) .unwrap(); - let mut metadata = - crate::http_cache::Metadata::read(&cache_filename).unwrap(); + let mut metadata = CachedUrlMetadata::read(&cache_filename).unwrap(); metadata.headers = HashMap::new(); metadata .headers @@ -1265,8 +1140,7 @@ mod tests { .await; assert!(result.is_ok()); - let metadata_filename = - crate::http_cache::Metadata::filename(&cache_filename); + let metadata_filename = CachedUrlMetadata::filename(&cache_filename); let metadata_file = fs::File::open(metadata_filename).unwrap(); let metadata_file_metadata = metadata_file.metadata().unwrap(); let metadata_file_modified_01 = metadata_file_metadata.modified().unwrap(); @@ -1285,8 +1159,7 @@ mod tests { .await; assert!(result.is_ok()); - let metadata_filename = - crate::http_cache::Metadata::filename(&cache_filename); + let metadata_filename = CachedUrlMetadata::filename(&cache_filename); let metadata_file = fs::File::open(metadata_filename).unwrap(); let metadata_file_metadata = metadata_file.metadata().unwrap(); let metadata_file_modified_02 = metadata_file_metadata.modified().unwrap(); @@ -1438,7 +1311,7 @@ mod tests { assert!(result.is_ok()); let metadata_filename = - crate::http_cache::Metadata::filename(&redirected_cache_filename); + CachedUrlMetadata::filename(&redirected_cache_filename); let metadata_file = fs::File::open(metadata_filename).unwrap(); let metadata_file_metadata = metadata_file.metadata().unwrap(); let metadata_file_modified_01 = metadata_file_metadata.modified().unwrap(); @@ -1458,7 +1331,7 @@ mod tests { assert!(result.is_ok()); let metadata_filename = - crate::http_cache::Metadata::filename(&redirected_cache_filename); + CachedUrlMetadata::filename(&redirected_cache_filename); let metadata_file = fs::File::open(metadata_filename).unwrap(); let metadata_file_metadata = metadata_file.metadata().unwrap(); let metadata_file_modified_02 = metadata_file_metadata.modified().unwrap(); diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index c4512a8039..7f7f698718 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -1,6 +1,7 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use crate::http_cache; +use crate::cache::CachedUrlMetadata; +use crate::cache::HttpCache; use deno_core::parking_lot::Mutex; use deno_core::ModuleSpecifier; @@ -49,14 +50,14 @@ struct Metadata { #[derive(Debug, Default, Clone)] pub struct CacheMetadata { - cache: http_cache::HttpCache, + cache: HttpCache, metadata: Arc>>, } impl CacheMetadata { pub fn new(location: &Path) -> Self { Self { - cache: http_cache::HttpCache::new(location), + cache: HttpCache::new(location), metadata: Default::default(), } } @@ -87,8 +88,7 @@ impl CacheMetadata { return None; } let cache_filename = self.cache.get_cache_filename(specifier)?; - let specifier_metadata = - http_cache::Metadata::read(&cache_filename).ok()?; + let specifier_metadata = CachedUrlMetadata::read(&cache_filename).ok()?; let values = Arc::new(parse_metadata(&specifier_metadata.headers)); let version = calculate_fs_version(&cache_filename); let mut metadata_map = self.metadata.lock(); @@ -98,7 +98,7 @@ impl CacheMetadata { } pub fn set_location(&mut self, location: &Path) { - self.cache = http_cache::HttpCache::new(location); + self.cache = HttpCache::new(location); self.metadata.lock().clear(); } } diff --git a/cli/lsp/completions.rs b/cli/lsp/completions.rs index 67978fbc98..5e0fad0f43 100644 --- a/cli/lsp/completions.rs +++ b/cli/lsp/completions.rs @@ -7,9 +7,9 @@ use super::lsp_custom; use super::registries::ModuleRegistry; use super::tsc; -use crate::fs_util::is_supported_ext; -use crate::fs_util::relative_specifier; -use crate::fs_util::specifier_to_file_path; +use crate::util::path::is_supported_ext; +use crate::util::path::relative_specifier; +use crate::util::path::specifier_to_file_path; use deno_ast::LineAndColumnIndex; use deno_ast::SourceTextInfo; @@ -505,7 +505,7 @@ fn get_workspace_completions( #[cfg(test)] mod tests { use super::*; - use crate::http_cache::HttpCache; + use crate::cache::HttpCache; use crate::lsp::documents::Documents; use crate::lsp::documents::LanguageId; use deno_core::resolve_url; diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 04a48435f1..8d5da46b7a 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -2,7 +2,8 @@ use super::client::Client; use super::logging::lsp_log; -use crate::fs_util; +use crate::util::path::ensure_directory_specifier; +use crate::util::path::specifier_to_file_path; use deno_core::error::AnyError; use deno_core::serde::Deserialize; use deno_core::serde::Serialize; @@ -549,11 +550,11 @@ impl Config { workspace: &ModuleSpecifier, enabled_paths: Vec, ) -> bool { - let workspace = fs_util::ensure_directory_specifier(workspace.clone()); + let workspace = ensure_directory_specifier(workspace.clone()); let key = workspace.to_string(); let mut touched = false; if !enabled_paths.is_empty() { - if let Ok(workspace_path) = fs_util::specifier_to_file_path(&workspace) { + if let Ok(workspace_path) = specifier_to_file_path(&workspace) { let mut paths = Vec::new(); for path in &enabled_paths { let fs_path = workspace_path.join(path); diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index 5c1e60e736..ce8fead0d6 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -6,12 +6,11 @@ use super::tsc; use super::tsc::AssetDocument; use crate::args::ConfigFile; +use crate::cache::CachedUrlMetadata; +use crate::cache::HttpCache; use crate::file_fetcher::get_source_from_bytes; use crate::file_fetcher::map_content_type; use crate::file_fetcher::SUPPORTED_SCHEMES; -use crate::fs_util::specifier_to_file_path; -use crate::http_cache; -use crate::http_cache::HttpCache; use crate::node; use crate::node::node_resolve_npm_reference; use crate::node::NodeResolution; @@ -20,7 +19,8 @@ use crate::npm::NpmPackageReference; use crate::npm::NpmPackageReq; use crate::npm::NpmPackageResolver; use crate::resolver::CliResolver; -use crate::text_encoding; +use crate::util::path::specifier_to_file_path; +use crate::util::text_encoding; use deno_ast::MediaType; use deno_ast::ParsedSource; @@ -610,7 +610,7 @@ impl SpecifierResolver { ) -> Option { let cache_filename = self.cache.get_cache_filename(specifier)?; if redirect_limit > 0 && cache_filename.is_file() { - let headers = http_cache::Metadata::read(&cache_filename) + let headers = CachedUrlMetadata::read(&cache_filename) .ok() .map(|m| m.headers)?; if let Some(location) = headers.get("location") { @@ -657,8 +657,7 @@ impl FileSystemDocuments { ) } else { let cache_filename = cache.get_cache_filename(specifier)?; - let specifier_metadata = - http_cache::Metadata::read(&cache_filename).ok()?; + let specifier_metadata = CachedUrlMetadata::read(&cache_filename).ok()?; let maybe_content_type = specifier_metadata.headers.get("content-type").cloned(); let maybe_headers = Some(&specifier_metadata.headers); diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 081bbf4296..11897af9d9 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -57,6 +57,8 @@ use super::tsc::Assets; use super::tsc::AssetsSnapshot; use super::tsc::TsServer; use super::urls; +use crate::args::get_root_cert_store; +use crate::args::CacheSetting; use crate::args::CliOptions; use crate::args::ConfigFile; use crate::args::Flags; @@ -64,10 +66,7 @@ use crate::args::FmtConfig; use crate::args::LintConfig; use crate::args::TsConfig; use crate::cache::DenoDir; -use crate::file_fetcher::get_root_cert_store; use crate::file_fetcher::get_source_from_data_url; -use crate::file_fetcher::CacheSetting; -use crate::fs_util; use crate::graph_util::graph_valid; use crate::http_util::HttpClient; use crate::npm::NpmCache; @@ -75,9 +74,12 @@ use crate::npm::NpmPackageResolver; use crate::npm::RealNpmRegistryApi; use crate::proc_state::import_map_from_text; use crate::proc_state::ProcState; -use crate::progress_bar::ProgressBar; use crate::tools::fmt::format_file; use crate::tools::fmt::format_parsed_source; +use crate::util::fs::remove_dir_all_if_exists; +use crate::util::path::ensure_directory_specifier; +use crate::util::path::specifier_to_file_path; +use crate::util::progress_bar::ProgressBar; #[derive(Debug, Clone)] pub struct LanguageServer(Arc>); @@ -407,7 +409,7 @@ impl Inner { // file open and not a workspace. In those situations we can't // automatically discover the configuration if let Some(root_uri) = &self.config.root_uri { - let root_path = fs_util::specifier_to_file_path(root_uri)?; + let root_path = specifier_to_file_path(root_uri)?; let mut checked = std::collections::HashSet::new(); let maybe_config = ConfigFile::discover_from(&root_path, &mut checked)?; Ok(maybe_config.map(|c| { @@ -481,7 +483,7 @@ impl Inner { let cache_url = if let Ok(url) = Url::from_file_path(cache_str) { Ok(url) } else if let Some(root_uri) = &self.config.root_uri { - let root_path = fs_util::specifier_to_file_path(root_uri)?; + let root_path = specifier_to_file_path(root_uri)?; let cache_path = root_path.join(cache_str); Url::from_file_path(cache_path).map_err(|_| { anyhow!("Bad file path for import path: {:?}", cache_str) @@ -492,7 +494,7 @@ impl Inner { cache_str )) }?; - let cache_path = fs_util::specifier_to_file_path(&cache_url)?; + let cache_path = specifier_to_file_path(&cache_url)?; lsp_log!( " Resolved cache path: \"{}\"", cache_path.to_string_lossy() @@ -521,7 +523,7 @@ impl Inner { .config .root_uri .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); + .and_then(|uri| specifier_to_file_path(uri).ok()); let root_cert_store = Some(get_root_cert_store( maybe_root_path, workspace_settings.certificate_stores.clone(), @@ -569,7 +571,7 @@ impl Inner { anyhow!("Bad data url for import map: {}", import_map_str) })?) } else if let Some(root_uri) = &self.config.root_uri { - let root_path = fs_util::specifier_to_file_path(root_uri)?; + let root_path = specifier_to_file_path(root_uri)?; let import_map_path = root_path.join(&import_map_str); Some(Url::from_file_path(import_map_path).map_err(|_| { anyhow!("Bad file path for import map: {}", import_map_str) @@ -612,7 +614,7 @@ impl Inner { let import_map_json = if import_map_url.scheme() == "data" { get_source_from_data_url(&import_map_url)?.0 } else { - let import_map_path = fs_util::specifier_to_file_path(&import_map_url)?; + let import_map_path = specifier_to_file_path(&import_map_url)?; lsp_log!( " Resolved import map: \"{}\"", import_map_path.to_string_lossy() @@ -768,7 +770,7 @@ impl Inner { self.config.root_uri = params .root_uri .map(|s| self.url_map.normalize_url(&s)) - .map(fs_util::ensure_directory_specifier); + .map(ensure_directory_specifier); if let Some(value) = params.initialization_options { self.config.set_workspace_settings(value).map_err(|err| { @@ -1137,11 +1139,10 @@ impl Inner { _ => return Ok(None), }; let mark = self.performance.mark("formatting", Some(¶ms)); - let file_path = - fs_util::specifier_to_file_path(&specifier).map_err(|err| { - error!("{}", err); - LspError::invalid_request() - })?; + let file_path = specifier_to_file_path(&specifier).map_err(|err| { + error!("{}", err); + LspError::invalid_request() + })?; let fmt_options = if let Some(fmt_config) = self.maybe_fmt_config.as_ref() { // skip formatting any files ignored by the config file @@ -2063,7 +2064,7 @@ impl Inner { .config .root_uri .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); + .and_then(|uri| specifier_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in incoming_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_incoming_call( @@ -2109,7 +2110,7 @@ impl Inner { .config .root_uri .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); + .and_then(|uri| specifier_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); for item in outgoing_calls.iter() { if let Some(resolved) = item.try_resolve_call_hierarchy_outgoing_call( @@ -2162,7 +2163,7 @@ impl Inner { .config .root_uri .as_ref() - .and_then(|uri| fs_util::specifier_to_file_path(uri).ok()); + .and_then(|uri| specifier_to_file_path(uri).ok()); let mut resolved_items = Vec::::new(); match one_or_many { tsc::OneOrMany::One(item) => { @@ -3010,7 +3011,7 @@ impl Inner { } async fn reload_import_registries(&mut self) -> LspResult> { - fs_util::remove_dir_all_if_exists(&self.module_registries_location) + remove_dir_all_if_exists(&self.module_registries_location) .await .map_err(|err| { error!("Unable to remove registries cache: {}", err); diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index 43500e6970..1488077dd4 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -12,10 +12,10 @@ use super::path_to_regex::StringOrNumber; use super::path_to_regex::StringOrVec; use super::path_to_regex::Token; +use crate::args::CacheSetting; use crate::cache::DenoDir; -use crate::file_fetcher::CacheSetting; +use crate::cache::HttpCache; use crate::file_fetcher::FileFetcher; -use crate::http_cache::HttpCache; use crate::http_util::HttpClient; use deno_core::anyhow::anyhow; diff --git a/cli/lsp/testing/definitions.rs b/cli/lsp/testing/definitions.rs index 14ac165fd1..a2cd780123 100644 --- a/cli/lsp/testing/definitions.rs +++ b/cli/lsp/testing/definitions.rs @@ -2,9 +2,9 @@ use super::lsp_custom; -use crate::checksum; use crate::lsp::analysis::source_range_to_lsp_range; use crate::lsp::client::TestingNotification; +use crate::util::checksum; use deno_ast::SourceRange; use deno_ast::SourceTextInfo; diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index d839cda568..bd4f076219 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -6,7 +6,6 @@ use super::lsp_custom; use crate::args::flags_from_vec; use crate::args::DenoSubcommand; -use crate::checksum; use crate::lsp::client::Client; use crate::lsp::client::TestingNotification; use crate::lsp::config; @@ -15,6 +14,7 @@ use crate::ops; use crate::proc_state; use crate::tools::test; use crate::tools::test::TestEventSender; +use crate::util::checksum; use crate::worker::create_main_worker_for_test_or_bench; use deno_core::anyhow::anyhow; diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 4bb5ae5f99..88de60131c 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -18,10 +18,10 @@ use super::urls::LspUrlMap; use super::urls::INVALID_SPECIFIER; use crate::args::TsConfig; -use crate::fs_util::relative_specifier; -use crate::fs_util::specifier_to_file_path; use crate::tsc; use crate::tsc::ResolveArgs; +use crate::util::path::relative_specifier; +use crate::util::path::specifier_to_file_path; use deno_core::anyhow::anyhow; use deno_core::error::custom_error; @@ -3445,7 +3445,7 @@ pub fn request( #[cfg(test)] mod tests { use super::*; - use crate::http_cache::HttpCache; + use crate::cache::HttpCache; use crate::http_util::HeadersMap; use crate::lsp::config::WorkspaceSettings; use crate::lsp::documents::Documents; diff --git a/cli/lsp/urls.rs b/cli/lsp/urls.rs index 9b14098ae8..5aed54ad5e 100644 --- a/cli/lsp/urls.rs +++ b/cli/lsp/urls.rs @@ -56,7 +56,7 @@ fn hash_data_specifier(specifier: &ModuleSpecifier) -> String { file_name_str.push('?'); file_name_str.push_str(query); } - crate::checksum::gen(&[file_name_str.as_bytes()]) + crate::util::checksum::gen(&[file_name_str.as_bytes()]) } #[derive(Debug, Default)] diff --git a/cli/main.rs b/cli/main.rs index 4eaaeb755a..ad585c4158 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -3,20 +3,13 @@ mod args; mod auth_tokens; mod cache; -mod checksum; mod deno_std; -mod diff; -mod display; mod emit; mod errors; mod file_fetcher; -mod file_watcher; -mod fs_util; mod graph_util; -mod http_cache; mod http_util; mod js; -mod logger; mod lsp; mod module_loader; mod napi; @@ -24,15 +17,12 @@ mod node; mod npm; mod ops; mod proc_state; -mod progress_bar; mod resolver; mod standalone; -mod text_encoding; mod tools; mod tsc; -mod unix_util; +mod util; mod version; -mod windows_util; mod worker; use crate::args::flags_from_vec; @@ -63,11 +53,12 @@ use crate::args::UpgradeFlags; use crate::args::VendorFlags; use crate::cache::TypeCheckCache; use crate::file_fetcher::File; -use crate::file_watcher::ResolutionResult; use crate::graph_util::graph_lock_or_exit; use crate::proc_state::ProcState; use crate::resolver::CliResolver; use crate::tools::check; +use crate::util::display; +use crate::util::file_watcher::ResolutionResult; use args::CliOptions; use args::Lockfile; @@ -482,7 +473,7 @@ async fn bundle_command( if let Some(out_file) = out_file.as_ref() { let output_bytes = bundle_output.code.as_bytes(); let output_len = output_bytes.len(); - fs_util::write_file(out_file, output_bytes, 0o644)?; + util::fs::write_file(out_file, output_bytes, 0o644)?; info!( "{} {:?} ({})", colors::green("Emit"), @@ -498,7 +489,7 @@ async fn bundle_command( "map".to_string() }; let map_out_file = out_file.with_extension(ext); - fs_util::write_file(&map_out_file, map_bytes, 0o644)?; + util::fs::write_file(&map_out_file, map_bytes, 0o644)?; info!( "{} {:?} ({})", colors::green("Emit"), @@ -515,10 +506,10 @@ async fn bundle_command( }; if cli_options.watch_paths().is_some() { - file_watcher::watch_func( + util::file_watcher::watch_func( resolver, operation, - file_watcher::PrintConfig { + util::file_watcher::PrintConfig { job_name: "Bundle".to_string(), clear_screen: !cli_options.no_clear_screen(), }, @@ -660,11 +651,11 @@ async fn run_with_watch(flags: Flags, script: String) -> Result { }) }; - file_watcher::watch_func2( + util::file_watcher::watch_func2( receiver, operation, (sender, main_module), - file_watcher::PrintConfig { + util::file_watcher::PrintConfig { job_name: "Process".to_string(), clear_screen: !flags.no_clear_screen, }, @@ -952,8 +943,8 @@ fn unwrap_or_exit(result: Result) -> T { pub fn main() { setup_panic_hook(); - unix_util::raise_fd_limit(); - windows_util::ensure_stdio_open(); + util::unix::raise_fd_limit(); + util::windows::ensure_stdio_open(); #[cfg(windows)] colors::enable_ansi(); // For Windows 10 @@ -984,7 +975,7 @@ pub fn main() { init_v8_flags(&flags.v8_flags); } - logger::init(flags.log_level); + util::logger::init(flags.log_level); get_subcommand(flags).await }; diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 41ec7e28ce..6adfc9f491 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -5,8 +5,8 @@ use crate::emit::emit_parsed_source; use crate::graph_util::ModuleEntry; use crate::node; use crate::proc_state::ProcState; -use crate::text_encoding::code_without_source_map; -use crate::text_encoding::source_map_from_code; +use crate::util::text_encoding::code_without_source_map; +use crate::util::text_encoding::source_map_from_code; use deno_ast::MediaType; use deno_core::anyhow::anyhow; diff --git a/cli/napi_sym/Cargo.toml b/cli/napi/sym/Cargo.toml similarity index 100% rename from cli/napi_sym/Cargo.toml rename to cli/napi/sym/Cargo.toml diff --git a/cli/napi_sym/README.md b/cli/napi/sym/README.md similarity index 100% rename from cli/napi_sym/README.md rename to cli/napi/sym/README.md diff --git a/cli/napi_sym/lib.rs b/cli/napi/sym/lib.rs similarity index 95% rename from cli/napi_sym/lib.rs rename to cli/napi/sym/lib.rs index 2805c99572..984d7f4bcd 100644 --- a/cli/napi_sym/lib.rs +++ b/cli/napi/sym/lib.rs @@ -20,7 +20,7 @@ pub fn napi_sym(_attr: TokenStream, item: TokenStream) -> TokenStream { let name = &func.sig.ident; assert!( exports.symbols.contains(&name.to_string()), - "tools/napi/symbol_exports.json is out of sync!" + "tools/napi/sym/symbol_exports.json is out of sync!" ); let block = &func.block; diff --git a/cli/napi_sym/symbol_exports.json b/cli/napi/sym/symbol_exports.json similarity index 100% rename from cli/napi_sym/symbol_exports.json rename to cli/napi/sym/symbol_exports.json diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 5e2f06ef74..ad6ab9db2c 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -14,11 +14,13 @@ use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::url::Url; +use crate::args::CacheSetting; use crate::cache::DenoDir; -use crate::file_fetcher::CacheSetting; -use crate::fs_util; use crate::http_util::HttpClient; -use crate::progress_bar::ProgressBar; +use crate::util::fs::canonicalize_path; +use crate::util::fs::hard_link_dir_recursive; +use crate::util::path::root_url_to_safe_local_dirname; +use crate::util::progress_bar::ProgressBar; use super::registry::NpmPackageVersionDistInfo; use super::semver::NpmVersion; @@ -162,7 +164,7 @@ impl ReadonlyNpmCache { std::fs::create_dir_all(root_dir) .with_context(|| format!("Error creating {}", root_dir.display()))?; } - Ok(crate::fs_util::canonicalize_path(root_dir)?) + Ok(canonicalize_path(root_dir)?) } // this may fail on readonly file systems, so just ignore if so @@ -227,7 +229,7 @@ impl ReadonlyNpmCache { pub fn registry_folder(&self, registry_url: &Url) -> PathBuf { self .root_dir - .join(fs_util::root_url_to_safe_local_dirname(registry_url)) + .join(root_url_to_safe_local_dirname(registry_url)) } pub fn resolve_package_folder_id_from_specifier( @@ -252,7 +254,7 @@ impl ReadonlyNpmCache { .root_dir_url .join(&format!( "{}/", - fs_util::root_url_to_safe_local_dirname(registry_url) + root_url_to_safe_local_dirname(registry_url) .to_string_lossy() .replace('\\', "/") )) @@ -457,12 +459,7 @@ impl NpmCache { with_folder_sync_lock( (id.name.as_str(), &id.version), &package_folder, - || { - fs_util::hard_link_dir_recursive( - &original_package_folder, - &package_folder, - ) - }, + || hard_link_dir_recursive(&original_package_folder, &package_folder), )?; Ok(()) } diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index c62e6e1e7d..9ba5653662 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -21,11 +21,11 @@ use deno_core::url::Url; use deno_runtime::colors; use serde::Serialize; -use crate::file_fetcher::CacheSetting; -use crate::fs_util; -use crate::http_cache::CACHE_PERM; +use crate::args::CacheSetting; +use crate::cache::CACHE_PERM; use crate::http_util::HttpClient; -use crate::progress_bar::ProgressBar; +use crate::util::fs::atomic_write_file; +use crate::util::progress_bar::ProgressBar; use super::cache::NpmCache; use super::resolution::NpmVersionMatcher; @@ -405,7 +405,7 @@ impl RealNpmRegistryApiInner { let file_cache_path = self.get_package_file_cache_path(name); let file_text = serde_json::to_string(&package_info)?; std::fs::create_dir_all(file_cache_path.parent().unwrap())?; - fs_util::atomic_write_file(&file_cache_path, file_text, CACHE_PERM)?; + atomic_write_file(&file_cache_path, file_text, CACHE_PERM)?; Ok(()) } diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 044c889d8f..46cfec48fb 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -16,7 +16,6 @@ use deno_runtime::deno_node::PackageJson; use deno_runtime::deno_node::TYPES_CONDITIONS; use crate::args::Lockfile; -use crate::fs_util; use crate::npm::resolution::NpmResolution; use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::resolvers::common::cache_packages; @@ -125,7 +124,7 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { fn package_size(&self, package_id: &NpmPackageId) -> Result { let package_folder = self.package_folder(package_id); - Ok(fs_util::dir_size(&package_folder)?) + Ok(crate::util::fs::dir_size(&package_folder)?) } fn has_packages(&self) -> bool { diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index ff699f26f3..3a9e97433b 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -10,6 +10,7 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use crate::util::fs::symlink_dir; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -23,7 +24,6 @@ use deno_runtime::deno_node::TYPES_CONDITIONS; use tokio::task::JoinHandle; use crate::args::Lockfile; -use crate::fs_util; use crate::npm::cache::mixed_case_package_name_encode; use crate::npm::cache::should_sync_download; use crate::npm::cache::NpmPackageCacheFolderId; @@ -34,6 +34,8 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; use crate::npm::RealNpmRegistryApi; +use crate::util::fs::copy_dir_recursive; +use crate::util::fs::hard_link_dir_recursive; use super::common::ensure_registry_read_permission; use super::common::types_package_name; @@ -203,7 +205,7 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { fn package_size(&self, package_id: &NpmPackageId) -> Result { let package_folder_path = self.get_package_id_folder(package_id)?; - Ok(fs_util::dir_size(&package_folder_path)?) + Ok(crate::util::fs::dir_size(&package_folder_path)?) } fn has_packages(&self) -> bool { @@ -318,7 +320,7 @@ async fn sync_resolution_with_fs( ®istry_url, ); // for now copy, but in the future consider hard linking - fs_util::copy_dir_recursive(&cache_folder, &package_path)?; + copy_dir_recursive(&cache_folder, &package_path)?; // write out a file that indicates this folder has been initialized fs::write(initialized_file, "")?; Ok(()) @@ -356,7 +358,7 @@ async fn sync_resolution_with_fs( .join("node_modules"), &package.id.name, ); - fs_util::hard_link_dir_recursive(&source_path, &package_path)?; + hard_link_dir_recursive(&source_path, &package_path)?; // write out a file that indicates this folder has been initialized fs::write(initialized_file, "")?; } @@ -467,7 +469,7 @@ fn symlink_package_dir( #[cfg(windows)] return junction_or_symlink_dir(old_path, new_path); #[cfg(not(windows))] - fs_util::symlink_dir(old_path, new_path) + symlink_dir(old_path, new_path) } #[cfg(windows)] @@ -477,6 +479,7 @@ fn junction_or_symlink_dir( ) -> Result<(), AnyError> { // Use junctions because they're supported on ntfs file systems without // needing to elevate privileges on Windows + match junction::create(old_path, new_path) { Ok(()) => Ok(()), Err(junction_err) => { @@ -486,7 +489,7 @@ fn junction_or_symlink_dir( log::warn!("Error creating junction. {:#}", junction_err); } - match fs_util::symlink_dir(old_path, new_path) { + match symlink_dir(old_path, new_path) { Ok(()) => Ok(()), Err(symlink_err) => bail!( concat!( diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 869874c8b6..3cc695523c 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -23,7 +23,7 @@ use std::path::PathBuf; use std::sync::Arc; use crate::args::Lockfile; -use crate::fs_util; +use crate::util::fs::canonicalize_path_maybe_not_exists; use self::common::InnerNpmPackageResolver; use self::local::LocalNpmPackageResolver; @@ -187,7 +187,7 @@ impl NpmPackageResolver { let path = self .inner .resolve_package_folder_from_deno_module(pkg_req)?; - let path = fs_util::canonicalize_path_maybe_not_exists(&path)?; + let path = canonicalize_path_maybe_not_exists(&path)?; log::debug!("Resolved {} to {}", pkg_req, path.display()); Ok(path) } diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 3b7a975735..9fcac2fe07 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -11,6 +11,7 @@ use crate::cache; use crate::cache::DenoDir; use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; +use crate::cache::HttpCache; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; use crate::cache::TypeCheckCache; @@ -19,7 +20,6 @@ use crate::file_fetcher::FileFetcher; use crate::graph_util::graph_lock_or_exit; use crate::graph_util::GraphData; use crate::graph_util::ModuleEntry; -use crate::http_cache; use crate::http_util::HttpClient; use crate::node; use crate::node::NodeResolution; @@ -28,9 +28,9 @@ use crate::npm::NpmCache; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageResolver; use crate::npm::RealNpmRegistryApi; -use crate::progress_bar::ProgressBar; use crate::resolver::CliResolver; use crate::tools::check; +use crate::util::progress_bar::ProgressBar; use deno_ast::MediaType; use deno_core::anyhow::anyhow; @@ -153,7 +153,7 @@ impl ProcState { let compiled_wasm_module_store = CompiledWasmModuleStore::default(); let dir = cli_options.resolve_deno_dir()?; let deps_cache_location = dir.deps_folder_path(); - let http_cache = http_cache::HttpCache::new(&deps_cache_location); + let http_cache = HttpCache::new(&deps_cache_location); let root_cert_store = cli_options.resolve_root_cert_store()?; let cache_usage = cli_options.cache_setting(); let progress_bar = ProgressBar::default(); diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index a81c0a406e..eeda3ea1a2 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -4,16 +4,16 @@ use crate::args::BenchFlags; use crate::args::Flags; use crate::args::TypeCheckMode; use crate::colors; -use crate::file_watcher; -use crate::file_watcher::ResolutionResult; -use crate::fs_util::collect_specifiers; -use crate::fs_util::is_supported_bench_path; use crate::graph_util::contains_specifier; use crate::graph_util::graph_valid; use crate::ops; use crate::proc_state::ProcState; use crate::tools::test::format_test_error; use crate::tools::test::TestFilter; +use crate::util::file_watcher; +use crate::util::file_watcher::ResolutionResult; +use crate::util::fs::collect_specifiers; +use crate::util::path::is_supported_ext; use crate::worker::create_main_worker_for_test_or_bench; use deno_core::error::generic_error; @@ -32,6 +32,7 @@ use log::Level; use serde::Deserialize; use serde::Serialize; use std::collections::HashSet; +use std::path::Path; use std::path::PathBuf; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedSender; @@ -469,6 +470,19 @@ async fn bench_specifiers( Ok(()) } +/// Checks if the path has a basename and extension Deno supports for benches. +fn is_supported_bench_path(path: &Path) -> bool { + if let Some(name) = path.file_stem() { + let basename = name.to_string_lossy(); + (basename.ends_with("_bench") + || basename.ends_with(".bench") + || basename == "bench") + && is_supported_ext(path) + } else { + false + } +} + pub async fn run_benchmarks( flags: Flags, bench_flags: BenchFlags, diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index b76792b1e6..b9bbe14f24 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -3,10 +3,10 @@ use crate::args::CoverageFlags; use crate::args::Flags; use crate::colors; -use crate::fs_util::collect_files; use crate::proc_state::ProcState; -use crate::text_encoding::source_map_from_code; use crate::tools::fmt::format_json; +use crate::util::fs::collect_files; +use crate::util::text_encoding::source_map_from_code; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; diff --git a/cli/tools/fmt.rs b/cli/tools/fmt.rs index 9474411b7d..721937b8d5 100644 --- a/cli/tools/fmt.rs +++ b/cli/tools/fmt.rs @@ -12,13 +12,13 @@ use crate::args::FmtFlags; use crate::args::FmtOptionsConfig; use crate::args::ProseWrap; use crate::colors; -use crate::diff::diff; -use crate::file_watcher; -use crate::file_watcher::ResolutionResult; -use crate::fs_util::collect_files; -use crate::fs_util::get_extension; -use crate::fs_util::specifier_to_file_path; -use crate::text_encoding; +use crate::util::diff::diff; +use crate::util::file_watcher; +use crate::util::file_watcher::ResolutionResult; +use crate::util::fs::collect_files; +use crate::util::path::get_extension; +use crate::util::path::specifier_to_file_path; +use crate::util::text_encoding; use deno_ast::ParsedSource; use deno_core::anyhow::bail; use deno_core::anyhow::Context; diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 38aa40a5e3..a81dcb55c3 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -20,7 +20,6 @@ use deno_runtime::colors; use crate::args::Flags; use crate::args::InfoFlags; -use crate::checksum; use crate::display; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReference; @@ -29,6 +28,7 @@ use crate::npm::NpmPackageResolver; use crate::npm::NpmResolutionPackage; use crate::npm::NpmResolutionSnapshot; use crate::proc_state::ProcState; +use crate::util::checksum; pub async fn info(flags: Flags, info_flags: InfoFlags) -> Result<(), AnyError> { let ps = ProcState::build(flags).await?; diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index b964619d98..6914c99193 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -4,8 +4,8 @@ use crate::args::ConfigFlag; use crate::args::Flags; use crate::args::InstallFlags; use crate::args::TypeCheckMode; -use crate::fs_util; use crate::npm::NpmPackageReference; +use crate::util::fs::canonicalize_path_maybe_not_exists; use deno_core::anyhow::Context; use deno_core::error::generic_error; use deno_core::error::AnyError; @@ -107,9 +107,7 @@ exec deno {} "$@" fn get_installer_root() -> Result { if let Ok(env_dir) = env::var("DENO_INSTALL_ROOT") { if !env_dir.is_empty() { - return fs_util::canonicalize_path_maybe_not_exists(&PathBuf::from( - env_dir, - )); + return canonicalize_path_maybe_not_exists(&PathBuf::from(env_dir)); } } // Note: on Windows, the $HOME environment variable may be set by users or by @@ -167,7 +165,7 @@ pub fn infer_name_from_url(url: &Url) -> Option { pub fn uninstall(name: String, root: Option) -> Result<(), AnyError> { let root = if let Some(root) = root { - fs_util::canonicalize_path_maybe_not_exists(&root)? + canonicalize_path_maybe_not_exists(&root)? } else { get_installer_root()? }; @@ -275,7 +273,7 @@ fn resolve_shim_data( install_flags: &InstallFlags, ) -> Result { let root = if let Some(root) = &install_flags.root { - fs_util::canonicalize_path_maybe_not_exists(root)? + canonicalize_path_maybe_not_exists(root)? } else { get_installer_root()? }; @@ -450,6 +448,7 @@ mod tests { use super::*; use crate::args::ConfigFlag; + use crate::util::fs::canonicalize_path; use std::process::Command; use test_util::testdata_path; use test_util::TempDir; @@ -746,7 +745,7 @@ mod tests { #[test] fn install_npm_lockfile_default() { - let temp_dir = fs_util::canonicalize_path(&env::temp_dir()).unwrap(); + let temp_dir = canonicalize_path(&env::temp_dir()).unwrap(); let shim_data = resolve_shim_data( &Flags { allow_all: true, diff --git a/cli/tools/lint.rs b/cli/tools/lint.rs index bfbff8005e..2f7cd51111 100644 --- a/cli/tools/lint.rs +++ b/cli/tools/lint.rs @@ -10,13 +10,13 @@ use crate::args::Flags; use crate::args::LintConfig; use crate::args::LintFlags; use crate::colors; -use crate::file_watcher; -use crate::file_watcher::ResolutionResult; -use crate::fs_util::collect_files; -use crate::fs_util::is_supported_ext; -use crate::fs_util::specifier_to_file_path; use crate::proc_state::ProcState; use crate::tools::fmt::run_parallelized; +use crate::util::file_watcher; +use crate::util::file_watcher::ResolutionResult; +use crate::util::fs::collect_files; +use crate::util::path::is_supported_ext; +use crate::util::path::specifier_to_file_path; use deno_ast::MediaType; use deno_core::anyhow::anyhow; use deno_core::error::generic_error; diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index f173103cb2..4436aaa282 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -6,9 +6,9 @@ use crate::args::Flags; use crate::args::RunFlags; use crate::args::TypeCheckMode; use crate::cache::DenoDir; -use crate::fs_util; use crate::standalone::Metadata; use crate::standalone::MAGIC_TRAILER; +use crate::util::path::path_has_trailing_slash; use crate::ProcState; use deno_core::anyhow::bail; use deno_core::anyhow::Context; @@ -299,7 +299,7 @@ pub fn resolve_compile_executable_output_path( ) -> Result { let module_specifier = resolve_url_or_path(&compile_flags.source_file)?; compile_flags.output.as_ref().and_then(|output| { - if fs_util::path_has_trailing_slash(output) { + if path_has_trailing_slash(output) { let infer_file_name = infer_name_from_url(&module_specifier).map(PathBuf::from)?; Some(output.join(infer_file_name)) } else { diff --git a/cli/tools/task.rs b/cli/tools/task.rs index 4cbba8661d..39d493492f 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -3,8 +3,8 @@ use crate::args::Flags; use crate::args::TaskFlags; use crate::colors; -use crate::fs_util; use crate::proc_state::ProcState; +use crate::util::fs::canonicalize_path; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -40,7 +40,7 @@ pub async fn execute_script( } let cwd = match task_flags.cwd { - Some(path) => fs_util::canonicalize_path(&PathBuf::from(path))?, + Some(path) => canonicalize_path(&PathBuf::from(path))?, None => config_file_path.parent().unwrap().to_owned(), }; let task_name = task_flags.task; diff --git a/cli/tools/test.rs b/cli/tools/test.rs index 1bb891a1ea..07d3f250d8 100644 --- a/cli/tools/test.rs +++ b/cli/tools/test.rs @@ -3,20 +3,20 @@ use crate::args::Flags; use crate::args::TestFlags; use crate::args::TypeCheckMode; -use crate::checksum; use crate::colors; use crate::display; use crate::file_fetcher::File; -use crate::file_watcher; -use crate::file_watcher::ResolutionResult; -use crate::fs_util::collect_specifiers; -use crate::fs_util::is_supported_test_ext; -use crate::fs_util::is_supported_test_path; -use crate::fs_util::specifier_to_file_path; use crate::graph_util::contains_specifier; use crate::graph_util::graph_valid; use crate::ops; use crate::proc_state::ProcState; +use crate::util::checksum; +use crate::util::file_watcher; +use crate::util::file_watcher::ResolutionResult; +use crate::util::fs::collect_specifiers; +use crate::util::path::get_extension; +use crate::util::path::is_supported_ext; +use crate::util::path::specifier_to_file_path; use crate::worker::create_main_worker_for_test_or_bench; use deno_ast::swc::common::comments::CommentKind; @@ -51,6 +51,7 @@ use std::fmt::Write as _; use std::io::Read; use std::io::Write; use std::num::NonZeroUsize; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; @@ -1184,6 +1185,44 @@ async fn test_specifiers( Ok(()) } +/// Checks if the path has a basename and extension Deno supports for tests. +fn is_supported_test_path(path: &Path) -> bool { + if let Some(name) = path.file_stem() { + let basename = name.to_string_lossy(); + (basename.ends_with("_test") + || basename.ends_with(".test") + || basename == "test") + && is_supported_ext(path) + } else { + false + } +} + +/// Checks if the path has an extension Deno supports for tests. +fn is_supported_test_ext(path: &Path) -> bool { + if let Some(ext) = get_extension(path) { + matches!( + ext.as_str(), + "ts" + | "tsx" + | "js" + | "jsx" + | "mjs" + | "mts" + | "cjs" + | "cts" + | "md" + | "mkd" + | "mkdn" + | "mdwn" + | "mdown" + | "markdown" + ) + } else { + false + } +} + /// Collects specifiers marking them with the appropriate test mode while maintaining the natural /// input order. /// @@ -1667,3 +1706,67 @@ fn start_output_redirect_thread( } }); } + +#[cfg(test)] +mod inner_test { + use std::path::Path; + + use super::*; + + #[test] + fn test_is_supported_test_ext() { + assert!(!is_supported_test_ext(Path::new("tests/subdir/redirects"))); + assert!(is_supported_test_ext(Path::new("README.md"))); + assert!(is_supported_test_ext(Path::new("readme.MD"))); + assert!(is_supported_test_ext(Path::new("lib/typescript.d.ts"))); + assert!(is_supported_test_ext(Path::new( + "testdata/run/001_hello.js" + ))); + assert!(is_supported_test_ext(Path::new( + "testdata/run/002_hello.ts" + ))); + assert!(is_supported_test_ext(Path::new("foo.jsx"))); + assert!(is_supported_test_ext(Path::new("foo.tsx"))); + assert!(is_supported_test_ext(Path::new("foo.TS"))); + assert!(is_supported_test_ext(Path::new("foo.TSX"))); + assert!(is_supported_test_ext(Path::new("foo.JS"))); + assert!(is_supported_test_ext(Path::new("foo.JSX"))); + assert!(is_supported_test_ext(Path::new("foo.mjs"))); + assert!(is_supported_test_ext(Path::new("foo.mts"))); + assert!(is_supported_test_ext(Path::new("foo.cjs"))); + assert!(is_supported_test_ext(Path::new("foo.cts"))); + assert!(!is_supported_test_ext(Path::new("foo.mjsx"))); + assert!(!is_supported_test_ext(Path::new("foo.jsonc"))); + assert!(!is_supported_test_ext(Path::new("foo.JSONC"))); + assert!(!is_supported_test_ext(Path::new("foo.json"))); + assert!(!is_supported_test_ext(Path::new("foo.JsON"))); + } + + #[test] + fn test_is_supported_test_path() { + assert!(is_supported_test_path(Path::new( + "tests/subdir/foo_test.ts" + ))); + assert!(is_supported_test_path(Path::new( + "tests/subdir/foo_test.tsx" + ))); + assert!(is_supported_test_path(Path::new( + "tests/subdir/foo_test.js" + ))); + assert!(is_supported_test_path(Path::new( + "tests/subdir/foo_test.jsx" + ))); + assert!(is_supported_test_path(Path::new("bar/foo.test.ts"))); + assert!(is_supported_test_path(Path::new("bar/foo.test.tsx"))); + assert!(is_supported_test_path(Path::new("bar/foo.test.js"))); + assert!(is_supported_test_path(Path::new("bar/foo.test.jsx"))); + assert!(is_supported_test_path(Path::new("foo/bar/test.js"))); + assert!(is_supported_test_path(Path::new("foo/bar/test.jsx"))); + assert!(is_supported_test_path(Path::new("foo/bar/test.ts"))); + assert!(is_supported_test_path(Path::new("foo/bar/test.tsx"))); + assert!(!is_supported_test_path(Path::new("README.md"))); + assert!(!is_supported_test_path(Path::new("lib/typescript.d.ts"))); + assert!(!is_supported_test_path(Path::new("notatest.js"))); + assert!(!is_supported_test_path(Path::new("NotAtest.ts"))); + } +} diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs index 5435361283..d1152b12b4 100644 --- a/cli/tools/vendor/mappings.rs +++ b/cli/tools/vendor/mappings.rs @@ -13,8 +13,8 @@ use deno_graph::ModuleGraph; use deno_graph::Position; use deno_graph::Resolved; -use crate::fs_util::path_with_stem_suffix; -use crate::fs_util::relative_specifier; +use crate::util::path::path_with_stem_suffix; +use crate::util::path::relative_specifier; use super::specifiers::dir_name_for_root; use super::specifiers::get_unique_path; diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs index 3fd381b213..a1057d8383 100644 --- a/cli/tools/vendor/mod.rs +++ b/cli/tools/vendor/mod.rs @@ -15,11 +15,12 @@ use crate::args::CliOptions; use crate::args::Flags; use crate::args::FmtOptionsConfig; use crate::args::VendorFlags; -use crate::fs_util; -use crate::fs_util::relative_specifier; -use crate::fs_util::specifier_to_file_path; use crate::proc_state::ProcState; use crate::tools::fmt::format_json; +use crate::util::fs::canonicalize_path; +use crate::util::fs::resolve_from_cwd; +use crate::util::path::relative_specifier; +use crate::util::path::specifier_to_file_path; mod analyze; mod build; @@ -38,7 +39,7 @@ pub async fn vendor( Some(output_path) => output_path.to_owned(), None => PathBuf::from("vendor/"), }; - let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?; + let output_dir = resolve_from_cwd(&raw_output_dir)?; validate_output_dir(&output_dir, &vendor_flags)?; validate_options(&mut cli_options, &output_dir)?; let ps = ProcState::from_options(Arc::new(cli_options)).await?; @@ -110,18 +111,17 @@ fn validate_options( if let Some(import_map_path) = options .resolve_import_map_specifier()? .and_then(|p| specifier_to_file_path(&p).ok()) - .and_then(|p| fs_util::canonicalize_path(&p).ok()) + .and_then(|p| canonicalize_path(&p).ok()) { // make the output directory in order to canonicalize it for the check below std::fs::create_dir_all(output_dir)?; - let output_dir = - fs_util::canonicalize_path(output_dir).with_context(|| { - format!("Failed to canonicalize: {}", output_dir.display()) - })?; + let output_dir = canonicalize_path(output_dir).with_context(|| { + format!("Failed to canonicalize: {}", output_dir.display()) + })?; if import_map_path.starts_with(&output_dir) { // canonicalize to make the test for this pass on the CI - let cwd = fs_util::canonicalize_path(&std::env::current_dir()?)?; + let cwd = canonicalize_path(&std::env::current_dir()?)?; // We don't allow using the output directory to help generate the // new state because this may lead to cryptic error messages. log::warn!( diff --git a/cli/tools/vendor/specifiers.rs b/cli/tools/vendor/specifiers.rs index 5d4f982786..9d513cc92a 100644 --- a/cli/tools/vendor/specifiers.rs +++ b/cli/tools/vendor/specifiers.rs @@ -8,8 +8,9 @@ use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; use deno_core::error::AnyError; -use crate::fs_util; -use crate::fs_util::path_with_stem_suffix; +use crate::util::path::is_banned_path_char; +use crate::util::path::path_with_stem_suffix; +use crate::util::path::root_url_to_safe_local_dirname; /// Partitions the provided specifiers by the non-path and non-query parts of a specifier. pub fn partition_by_root_specifiers<'a>( @@ -30,7 +31,7 @@ pub fn partition_by_root_specifiers<'a>( /// Gets the directory name to use for the provided root. pub fn dir_name_for_root(root: &ModuleSpecifier) -> PathBuf { - fs_util::root_url_to_safe_local_dirname(root) + root_url_to_safe_local_dirname(root) } /// Gets a unique file path given the provided file path @@ -74,13 +75,7 @@ pub fn is_remote_specifier_text(text: &str) -> bool { pub fn sanitize_filepath(text: &str) -> String { text .chars() - .map(|c| { - if fs_util::is_banned_path_char(c) { - '_' - } else { - c - } - }) + .map(|c| if is_banned_path_char(c) { '_' } else { c }) .collect() } diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index a8cb7bcabc..38ec2d263a 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -9,6 +9,7 @@ use crate::node::NodeResolution; use crate::node::NodeResolutionMode; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageResolver; +use crate::util::checksum; use deno_ast::MediaType; use deno_core::anyhow::anyhow; @@ -178,7 +179,7 @@ fn get_maybe_hash( if let Some(source) = maybe_source { let mut data = vec![source.as_bytes().to_owned()]; data.extend_from_slice(hash_data); - Some(crate::checksum::gen(&data)) + Some(checksum::gen(&data)) } else { None } @@ -186,7 +187,7 @@ fn get_maybe_hash( /// Hash the URL so it can be sent to `tsc` in a supportable way fn hash_url(specifier: &ModuleSpecifier, media_type: MediaType) -> String { - let hash = crate::checksum::gen(&[specifier.path().as_bytes()]); + let hash = checksum::gen(&[specifier.path().as_bytes()]); format!( "{}:///{}{}", specifier.scheme(), @@ -365,7 +366,7 @@ fn op_create_hash(s: &mut OpState, args: Value) -> Result { .context("Invalid request from JavaScript for \"op_create_hash\".")?; let mut data = vec![v.data.as_bytes().to_owned()]; data.extend_from_slice(&state.hash_data); - let hash = crate::checksum::gen(&data); + let hash = checksum::gen(&data); Ok(json!({ "hash": hash })) } diff --git a/cli/checksum.rs b/cli/util/checksum.rs similarity index 100% rename from cli/checksum.rs rename to cli/util/checksum.rs diff --git a/cli/diff.rs b/cli/util/diff.rs similarity index 100% rename from cli/diff.rs rename to cli/util/diff.rs diff --git a/cli/display.rs b/cli/util/display.rs similarity index 100% rename from cli/display.rs rename to cli/util/display.rs diff --git a/cli/file_watcher.rs b/cli/util/file_watcher.rs similarity index 99% rename from cli/file_watcher.rs rename to cli/util/file_watcher.rs index 78a5e7a821..5158437a02 100644 --- a/cli/file_watcher.rs +++ b/cli/util/file_watcher.rs @@ -1,7 +1,7 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use crate::colors; -use crate::fs_util::canonicalize_path; +use crate::util::fs::canonicalize_path; use deno_core::error::AnyError; use deno_core::error::JsError; diff --git a/cli/fs_util.rs b/cli/util/fs.rs similarity index 52% rename from cli/fs_util.rs rename to cli/util/fs.rs index a27586da20..35cdae4faf 100644 --- a/cli/fs_util.rs +++ b/cli/util/fs.rs @@ -1,13 +1,11 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use deno_core::anyhow::Context; -use deno_core::error::uri_error; use deno_core::error::AnyError; pub use deno_core::normalize_path; use deno_core::ModuleSpecifier; use deno_runtime::deno_crypto::rand; use deno_runtime::deno_node::PathClean; -use std::borrow::Cow; use std::env::current_dir; use std::fs::OpenOptions; use std::io::Error; @@ -18,6 +16,8 @@ use std::path::PathBuf; use std::time::Duration; use walkdir::WalkDir; +use super::path::specifier_to_file_path; + pub fn atomic_write_file>( filename: &Path, data: T, @@ -163,77 +163,6 @@ pub fn resolve_from_cwd(path: &Path) -> Result { Ok(normalize_path(&resolved_path)) } -/// Checks if the path has extension Deno supports. -pub fn is_supported_ext(path: &Path) -> bool { - if let Some(ext) = get_extension(path) { - matches!( - ext.as_str(), - "ts" | "tsx" | "js" | "jsx" | "mjs" | "mts" | "cjs" | "cts" - ) - } else { - false - } -} - -/// Checks if the path has a basename and extension Deno supports for tests. -pub fn is_supported_test_path(path: &Path) -> bool { - if let Some(name) = path.file_stem() { - let basename = name.to_string_lossy(); - (basename.ends_with("_test") - || basename.ends_with(".test") - || basename == "test") - && is_supported_ext(path) - } else { - false - } -} - -/// Checks if the path has a basename and extension Deno supports for benches. -pub fn is_supported_bench_path(path: &Path) -> bool { - if let Some(name) = path.file_stem() { - let basename = name.to_string_lossy(); - (basename.ends_with("_bench") - || basename.ends_with(".bench") - || basename == "bench") - && is_supported_ext(path) - } else { - false - } -} - -/// Checks if the path has an extension Deno supports for tests. -pub fn is_supported_test_ext(path: &Path) -> bool { - if let Some(ext) = get_extension(path) { - matches!( - ext.as_str(), - "ts" - | "tsx" - | "js" - | "jsx" - | "mjs" - | "mts" - | "cjs" - | "cts" - | "md" - | "mkd" - | "mkdn" - | "mdwn" - | "mdown" - | "markdown" - ) - } else { - false - } -} - -/// Get the extension of a file in lowercase. -pub fn get_extension(file_path: &Path) -> Option { - return file_path - .extension() - .and_then(|e| e.to_str()) - .map(|e| e.to_lowercase()); -} - /// Collects file paths that satisfy the given predicate, by recursively walking `files`. /// If the walker visits a path that is listed in `ignore`, it skips descending into the directory. pub fn collect_files

( @@ -461,201 +390,6 @@ pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> { Ok(()) } -/// Attempts to convert a specifier to a file path. By default, uses the Url -/// crate's `to_file_path()` method, but falls back to try and resolve unix-style -/// paths on Windows. -pub fn specifier_to_file_path( - specifier: &ModuleSpecifier, -) -> Result { - let result = if cfg!(windows) { - match specifier.to_file_path() { - Ok(path) => Ok(path), - Err(()) => { - // This might be a unix-style path which is used in the tests even on Windows. - // Attempt to see if we can convert it to a `PathBuf`. This code should be removed - // once/if https://github.com/servo/rust-url/issues/730 is implemented. - if specifier.scheme() == "file" - && specifier.host().is_none() - && specifier.port().is_none() - && specifier.path_segments().is_some() - { - let path_str = specifier.path(); - match String::from_utf8( - percent_encoding::percent_decode(path_str.as_bytes()).collect(), - ) { - Ok(path_str) => Ok(PathBuf::from(path_str)), - Err(_) => Err(()), - } - } else { - Err(()) - } - } - } - } else { - specifier.to_file_path() - }; - match result { - Ok(path) => Ok(path), - Err(()) => Err(uri_error(format!( - "Invalid file path.\n Specifier: {}", - specifier - ))), - } -} - -/// Ensures a specifier that will definitely be a directory has a trailing slash. -pub fn ensure_directory_specifier( - mut specifier: ModuleSpecifier, -) -> ModuleSpecifier { - let path = specifier.path(); - if !path.ends_with('/') { - let new_path = format!("{}/", path); - specifier.set_path(&new_path); - } - specifier -} - -/// Gets the parent of this module specifier. -pub fn specifier_parent(specifier: &ModuleSpecifier) -> ModuleSpecifier { - let mut specifier = specifier.clone(); - // don't use specifier.segments() because it will strip the leading slash - let mut segments = specifier.path().split('/').collect::>(); - if segments.iter().all(|s| s.is_empty()) { - return specifier; - } - if let Some(last) = segments.last() { - if last.is_empty() { - segments.pop(); - } - segments.pop(); - let new_path = format!("{}/", segments.join("/")); - specifier.set_path(&new_path); - } - specifier -} - -/// `from.make_relative(to)` but with fixes. -pub fn relative_specifier( - from: &ModuleSpecifier, - to: &ModuleSpecifier, -) -> Option { - let is_dir = to.path().ends_with('/'); - - if is_dir && from == to { - return Some("./".to_string()); - } - - // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged - let from = if !from.path().ends_with('/') { - if let Some(end_slash) = from.path().rfind('/') { - let mut new_from = from.clone(); - new_from.set_path(&from.path()[..end_slash + 1]); - Cow::Owned(new_from) - } else { - Cow::Borrowed(from) - } - } else { - Cow::Borrowed(from) - }; - - // workaround for url crate not adding a trailing slash for a directory - // it seems to be fixed once a version greater than 2.2.2 is released - let mut text = from.make_relative(to)?; - if is_dir && !text.ends_with('/') && to.query().is_none() { - text.push('/'); - } - - Some(if text.starts_with("../") || text.starts_with("./") { - text - } else { - format!("./{}", text) - }) -} - -/// This function checks if input path has trailing slash or not. If input path -/// has trailing slash it will return true else it will return false. -pub fn path_has_trailing_slash(path: &Path) -> bool { - if let Some(path_str) = path.to_str() { - if cfg!(windows) { - path_str.ends_with('\\') - } else { - path_str.ends_with('/') - } - } else { - false - } -} - -/// Gets a path with the specified file stem suffix. -/// -/// Ex. `file.ts` with suffix `_2` returns `file_2.ts` -pub fn path_with_stem_suffix(path: &Path, suffix: &str) -> PathBuf { - if let Some(file_name) = path.file_name().map(|f| f.to_string_lossy()) { - if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) { - if let Some(ext) = path.extension().map(|f| f.to_string_lossy()) { - return if file_stem.to_lowercase().ends_with(".d") { - path.with_file_name(format!( - "{}{}.{}.{}", - &file_stem[..file_stem.len() - ".d".len()], - suffix, - // maintain casing - &file_stem[file_stem.len() - "d".len()..], - ext - )) - } else { - path.with_file_name(format!("{}{}.{}", file_stem, suffix, ext)) - }; - } - } - - path.with_file_name(format!("{}{}", file_name, suffix)) - } else { - path.with_file_name(suffix) - } -} - -/// Gets if the provided character is not supported on all -/// kinds of file systems. -pub fn is_banned_path_char(c: char) -> bool { - matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') -} - -/// Gets a safe local directory name for the provided url. -/// -/// For example: -/// https://deno.land:8080/path -> deno.land_8080/path -pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf { - fn sanitize_segment(text: &str) -> String { - text - .chars() - .map(|c| if is_banned_segment_char(c) { '_' } else { c }) - .collect() - } - - fn is_banned_segment_char(c: char) -> bool { - matches!(c, '/' | '\\') || is_banned_path_char(c) - } - - let mut result = String::new(); - if let Some(domain) = root.domain() { - result.push_str(&sanitize_segment(domain)); - } - if let Some(port) = root.port() { - if !result.is_empty() { - result.push('_'); - } - result.push_str(&port.to_string()); - } - let mut result = PathBuf::from(result); - if let Some(segments) = root.path_segments() { - for segment in segments.filter(|s| !s.is_empty()) { - result = result.join(sanitize_segment(segment)); - } - } - - result -} - /// Gets the total size (in bytes) of a directory. pub fn dir_size(path: &Path) -> std::io::Result { let entries = std::fs::read_dir(path)?; @@ -718,83 +452,6 @@ mod tests { assert_eq!(resolve_from_cwd(expected).unwrap(), expected); } - #[test] - fn test_is_supported_ext() { - assert!(!is_supported_ext(Path::new("tests/subdir/redirects"))); - assert!(!is_supported_ext(Path::new("README.md"))); - assert!(is_supported_ext(Path::new("lib/typescript.d.ts"))); - assert!(is_supported_ext(Path::new("testdata/run/001_hello.js"))); - assert!(is_supported_ext(Path::new("testdata/run/002_hello.ts"))); - assert!(is_supported_ext(Path::new("foo.jsx"))); - assert!(is_supported_ext(Path::new("foo.tsx"))); - assert!(is_supported_ext(Path::new("foo.TS"))); - assert!(is_supported_ext(Path::new("foo.TSX"))); - assert!(is_supported_ext(Path::new("foo.JS"))); - assert!(is_supported_ext(Path::new("foo.JSX"))); - assert!(is_supported_ext(Path::new("foo.mjs"))); - assert!(is_supported_ext(Path::new("foo.mts"))); - assert!(is_supported_ext(Path::new("foo.cjs"))); - assert!(is_supported_ext(Path::new("foo.cts"))); - assert!(!is_supported_ext(Path::new("foo.mjsx"))); - } - - #[test] - fn test_is_supported_test_ext() { - assert!(!is_supported_test_ext(Path::new("tests/subdir/redirects"))); - assert!(is_supported_test_ext(Path::new("README.md"))); - assert!(is_supported_test_ext(Path::new("readme.MD"))); - assert!(is_supported_test_ext(Path::new("lib/typescript.d.ts"))); - assert!(is_supported_test_ext(Path::new( - "testdata/run/001_hello.js" - ))); - assert!(is_supported_test_ext(Path::new( - "testdata/run/002_hello.ts" - ))); - assert!(is_supported_test_ext(Path::new("foo.jsx"))); - assert!(is_supported_test_ext(Path::new("foo.tsx"))); - assert!(is_supported_test_ext(Path::new("foo.TS"))); - assert!(is_supported_test_ext(Path::new("foo.TSX"))); - assert!(is_supported_test_ext(Path::new("foo.JS"))); - assert!(is_supported_test_ext(Path::new("foo.JSX"))); - assert!(is_supported_test_ext(Path::new("foo.mjs"))); - assert!(is_supported_test_ext(Path::new("foo.mts"))); - assert!(is_supported_test_ext(Path::new("foo.cjs"))); - assert!(is_supported_test_ext(Path::new("foo.cts"))); - assert!(!is_supported_test_ext(Path::new("foo.mjsx"))); - assert!(!is_supported_test_ext(Path::new("foo.jsonc"))); - assert!(!is_supported_test_ext(Path::new("foo.JSONC"))); - assert!(!is_supported_test_ext(Path::new("foo.json"))); - assert!(!is_supported_test_ext(Path::new("foo.JsON"))); - } - - #[test] - fn test_is_supported_test_path() { - assert!(is_supported_test_path(Path::new( - "tests/subdir/foo_test.ts" - ))); - assert!(is_supported_test_path(Path::new( - "tests/subdir/foo_test.tsx" - ))); - assert!(is_supported_test_path(Path::new( - "tests/subdir/foo_test.js" - ))); - assert!(is_supported_test_path(Path::new( - "tests/subdir/foo_test.jsx" - ))); - assert!(is_supported_test_path(Path::new("bar/foo.test.ts"))); - assert!(is_supported_test_path(Path::new("bar/foo.test.tsx"))); - assert!(is_supported_test_path(Path::new("bar/foo.test.js"))); - assert!(is_supported_test_path(Path::new("bar/foo.test.jsx"))); - assert!(is_supported_test_path(Path::new("foo/bar/test.js"))); - assert!(is_supported_test_path(Path::new("foo/bar/test.jsx"))); - assert!(is_supported_test_path(Path::new("foo/bar/test.ts"))); - assert!(is_supported_test_path(Path::new("foo/bar/test.tsx"))); - assert!(!is_supported_test_path(Path::new("README.md"))); - assert!(!is_supported_test_path(Path::new("lib/typescript.d.ts"))); - assert!(!is_supported_test_path(Path::new("notatest.js"))); - assert!(!is_supported_test_path(Path::new("NotAtest.ts"))); - } - #[test] fn test_collect_files() { fn create_files(dir_path: &Path, files: &[&str]) { @@ -1001,209 +658,4 @@ mod tests { ); } } - - #[test] - fn test_specifier_to_file_path() { - run_success_test("file:///", "/"); - run_success_test("file:///test", "/test"); - run_success_test("file:///dir/test/test.txt", "/dir/test/test.txt"); - run_success_test( - "file:///dir/test%20test/test.txt", - "/dir/test test/test.txt", - ); - - fn run_success_test(specifier: &str, expected_path: &str) { - let result = - specifier_to_file_path(&ModuleSpecifier::parse(specifier).unwrap()) - .unwrap(); - assert_eq!(result, PathBuf::from(expected_path)); - } - } - - #[test] - fn test_ensure_directory_specifier() { - run_test("file:///", "file:///"); - run_test("file:///test", "file:///test/"); - run_test("file:///test/", "file:///test/"); - run_test("file:///test/other", "file:///test/other/"); - run_test("file:///test/other/", "file:///test/other/"); - - fn run_test(specifier: &str, expected: &str) { - let result = - ensure_directory_specifier(ModuleSpecifier::parse(specifier).unwrap()); - assert_eq!(result.to_string(), expected); - } - } - - #[test] - fn test_specifier_parent() { - run_test("file:///", "file:///"); - run_test("file:///test", "file:///"); - run_test("file:///test/", "file:///"); - run_test("file:///test/other", "file:///test/"); - run_test("file:///test/other.txt", "file:///test/"); - run_test("file:///test/other/", "file:///test/"); - - fn run_test(specifier: &str, expected: &str) { - let result = - specifier_parent(&ModuleSpecifier::parse(specifier).unwrap()); - assert_eq!(result.to_string(), expected); - } - } - - #[test] - fn test_relative_specifier() { - let fixtures: Vec<(&str, &str, Option<&str>)> = vec![ - ("file:///from", "file:///to", Some("./to")), - ("file:///from", "file:///from/other", Some("./from/other")), - ("file:///from", "file:///from/other/", Some("./from/other/")), - ("file:///from", "file:///other/from", Some("./other/from")), - ("file:///from/", "file:///other/from", Some("../other/from")), - ("file:///from", "file:///other/from/", Some("./other/from/")), - ( - "file:///from", - "file:///to/other.txt", - Some("./to/other.txt"), - ), - ( - "file:///from/test", - "file:///to/other.txt", - Some("../to/other.txt"), - ), - ( - "file:///from/other.txt", - "file:///to/other.txt", - Some("../to/other.txt"), - ), - ( - "https://deno.land/x/a/b/d.ts", - "https://deno.land/x/a/b/c.ts", - Some("./c.ts"), - ), - ( - "https://deno.land/x/a/b/d.ts", - "https://deno.land/x/a/c.ts", - Some("../c.ts"), - ), - ( - "https://deno.land/x/a/b/d.ts", - "https://deno.land/x/a/b/c/d.ts", - Some("./c/d.ts"), - ), - ( - "https://deno.land/x/a/b/c/", - "https://deno.land/x/a/b/c/d.ts", - Some("./d.ts"), - ), - ( - "https://deno.land/x/a/b/c/", - "https://deno.land/x/a/b/c/d/e.ts", - Some("./d/e.ts"), - ), - ( - "https://deno.land/x/a/b/c/f.ts", - "https://deno.land/x/a/b/c/d/e.ts", - Some("./d/e.ts"), - ), - ( - "https://deno.land/x/a/b/d.ts", - "https://deno.land/x/a/c.ts?foo=bar", - Some("../c.ts?foo=bar"), - ), - ( - "https://deno.land/x/a/b/d.ts?foo=bar", - "https://deno.land/x/a/b/c.ts", - Some("./c.ts"), - ), - ("file:///a/b/d.ts", "file:///a/b/c.ts", Some("./c.ts")), - ("https://deno.land/x/a/b/c.ts", "file:///a/b/c.ts", None), - ( - "https://deno.land/", - "https://deno.land/x/a/b/c.ts", - Some("./x/a/b/c.ts"), - ), - ( - "https://deno.land/x/d/e/f.ts", - "https://deno.land/x/a/b/c.ts", - Some("../../a/b/c.ts"), - ), - ]; - for (from_str, to_str, expected) in fixtures { - let from = ModuleSpecifier::parse(from_str).unwrap(); - let to = ModuleSpecifier::parse(to_str).unwrap(); - let actual = relative_specifier(&from, &to); - assert_eq!( - actual.as_deref(), - expected, - "from: \"{}\" to: \"{}\"", - from_str, - to_str - ); - } - } - - #[test] - fn test_path_has_trailing_slash() { - #[cfg(not(windows))] - { - run_test("/Users/johndoe/Desktop/deno-project/target/", true); - run_test(r"/Users/johndoe/deno-project/target//", true); - run_test("/Users/johndoe/Desktop/deno-project", false); - run_test(r"/Users/johndoe/deno-project\", false); - } - - #[cfg(windows)] - { - run_test(r"C:\test\deno-project\", true); - run_test(r"C:\test\deno-project\\", true); - run_test(r"C:\test\file.txt", false); - run_test(r"C:\test\file.txt/", false); - } - - fn run_test(path_str: &str, expected: bool) { - let path = Path::new(path_str); - let result = path_has_trailing_slash(path); - assert_eq!(result, expected); - } - } - - #[test] - fn test_path_with_stem_suffix() { - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/"), "_2"), - PathBuf::from("/_2") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test"), "_2"), - PathBuf::from("/test_2") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test.txt"), "_2"), - PathBuf::from("/test_2.txt") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test/subdir"), "_2"), - PathBuf::from("/test/subdir_2") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test/subdir.other.txt"), "_2"), - PathBuf::from("/test/subdir.other_2.txt") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test.d.ts"), "_2"), - PathBuf::from("/test_2.d.ts") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test.D.TS"), "_2"), - PathBuf::from("/test_2.D.TS") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test.d.mts"), "_2"), - PathBuf::from("/test_2.d.mts") - ); - assert_eq!( - path_with_stem_suffix(&PathBuf::from("/test.d.cts"), "_2"), - PathBuf::from("/test_2.d.cts") - ); - } } diff --git a/cli/logger.rs b/cli/util/logger.rs similarity index 100% rename from cli/logger.rs rename to cli/util/logger.rs diff --git a/cli/util/mod.rs b/cli/util/mod.rs new file mode 100644 index 0000000000..176991d32c --- /dev/null +++ b/cli/util/mod.rs @@ -0,0 +1,14 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +// Note: Only add code in this folder that has no application specific logic +pub mod checksum; +pub mod diff; +pub mod display; +pub mod file_watcher; +pub mod fs; +pub mod logger; +pub mod path; +pub mod progress_bar; +pub mod text_encoding; +pub mod unix; +pub mod windows; diff --git a/cli/util/path.rs b/cli/util/path.rs new file mode 100644 index 0000000000..6df982f4ed --- /dev/null +++ b/cli/util/path.rs @@ -0,0 +1,452 @@ +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; + +use deno_ast::ModuleSpecifier; +use deno_core::error::uri_error; +use deno_core::error::AnyError; + +/// Checks if the path has extension Deno supports. +pub fn is_supported_ext(path: &Path) -> bool { + if let Some(ext) = get_extension(path) { + matches!( + ext.as_str(), + "ts" | "tsx" | "js" | "jsx" | "mjs" | "mts" | "cjs" | "cts" + ) + } else { + false + } +} + +/// Get the extension of a file in lowercase. +pub fn get_extension(file_path: &Path) -> Option { + return file_path + .extension() + .and_then(|e| e.to_str()) + .map(|e| e.to_lowercase()); +} + +/// Attempts to convert a specifier to a file path. By default, uses the Url +/// crate's `to_file_path()` method, but falls back to try and resolve unix-style +/// paths on Windows. +pub fn specifier_to_file_path( + specifier: &ModuleSpecifier, +) -> Result { + let result = if cfg!(windows) { + match specifier.to_file_path() { + Ok(path) => Ok(path), + Err(()) => { + // This might be a unix-style path which is used in the tests even on Windows. + // Attempt to see if we can convert it to a `PathBuf`. This code should be removed + // once/if https://github.com/servo/rust-url/issues/730 is implemented. + if specifier.scheme() == "file" + && specifier.host().is_none() + && specifier.port().is_none() + && specifier.path_segments().is_some() + { + let path_str = specifier.path(); + match String::from_utf8( + percent_encoding::percent_decode(path_str.as_bytes()).collect(), + ) { + Ok(path_str) => Ok(PathBuf::from(path_str)), + Err(_) => Err(()), + } + } else { + Err(()) + } + } + } + } else { + specifier.to_file_path() + }; + match result { + Ok(path) => Ok(path), + Err(()) => Err(uri_error(format!( + "Invalid file path.\n Specifier: {}", + specifier + ))), + } +} + +/// Ensures a specifier that will definitely be a directory has a trailing slash. +pub fn ensure_directory_specifier( + mut specifier: ModuleSpecifier, +) -> ModuleSpecifier { + let path = specifier.path(); + if !path.ends_with('/') { + let new_path = format!("{}/", path); + specifier.set_path(&new_path); + } + specifier +} + +/// Gets the parent of this module specifier. +pub fn specifier_parent(specifier: &ModuleSpecifier) -> ModuleSpecifier { + let mut specifier = specifier.clone(); + // don't use specifier.segments() because it will strip the leading slash + let mut segments = specifier.path().split('/').collect::>(); + if segments.iter().all(|s| s.is_empty()) { + return specifier; + } + if let Some(last) = segments.last() { + if last.is_empty() { + segments.pop(); + } + segments.pop(); + let new_path = format!("{}/", segments.join("/")); + specifier.set_path(&new_path); + } + specifier +} + +/// `from.make_relative(to)` but with fixes. +pub fn relative_specifier( + from: &ModuleSpecifier, + to: &ModuleSpecifier, +) -> Option { + let is_dir = to.path().ends_with('/'); + + if is_dir && from == to { + return Some("./".to_string()); + } + + // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged + let from = if !from.path().ends_with('/') { + if let Some(end_slash) = from.path().rfind('/') { + let mut new_from = from.clone(); + new_from.set_path(&from.path()[..end_slash + 1]); + Cow::Owned(new_from) + } else { + Cow::Borrowed(from) + } + } else { + Cow::Borrowed(from) + }; + + // workaround for url crate not adding a trailing slash for a directory + // it seems to be fixed once a version greater than 2.2.2 is released + let mut text = from.make_relative(to)?; + if is_dir && !text.ends_with('/') && to.query().is_none() { + text.push('/'); + } + + Some(if text.starts_with("../") || text.starts_with("./") { + text + } else { + format!("./{}", text) + }) +} + +/// This function checks if input path has trailing slash or not. If input path +/// has trailing slash it will return true else it will return false. +pub fn path_has_trailing_slash(path: &Path) -> bool { + if let Some(path_str) = path.to_str() { + if cfg!(windows) { + path_str.ends_with('\\') + } else { + path_str.ends_with('/') + } + } else { + false + } +} + +/// Gets a path with the specified file stem suffix. +/// +/// Ex. `file.ts` with suffix `_2` returns `file_2.ts` +pub fn path_with_stem_suffix(path: &Path, suffix: &str) -> PathBuf { + if let Some(file_name) = path.file_name().map(|f| f.to_string_lossy()) { + if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) { + if let Some(ext) = path.extension().map(|f| f.to_string_lossy()) { + return if file_stem.to_lowercase().ends_with(".d") { + path.with_file_name(format!( + "{}{}.{}.{}", + &file_stem[..file_stem.len() - ".d".len()], + suffix, + // maintain casing + &file_stem[file_stem.len() - "d".len()..], + ext + )) + } else { + path.with_file_name(format!("{}{}.{}", file_stem, suffix, ext)) + }; + } + } + + path.with_file_name(format!("{}{}", file_name, suffix)) + } else { + path.with_file_name(suffix) + } +} + +/// Gets if the provided character is not supported on all +/// kinds of file systems. +pub fn is_banned_path_char(c: char) -> bool { + matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*') +} + +/// Gets a safe local directory name for the provided url. +/// +/// For example: +/// https://deno.land:8080/path -> deno.land_8080/path +pub fn root_url_to_safe_local_dirname(root: &ModuleSpecifier) -> PathBuf { + fn sanitize_segment(text: &str) -> String { + text + .chars() + .map(|c| if is_banned_segment_char(c) { '_' } else { c }) + .collect() + } + + fn is_banned_segment_char(c: char) -> bool { + matches!(c, '/' | '\\') || is_banned_path_char(c) + } + + let mut result = String::new(); + if let Some(domain) = root.domain() { + result.push_str(&sanitize_segment(domain)); + } + if let Some(port) = root.port() { + if !result.is_empty() { + result.push('_'); + } + result.push_str(&port.to_string()); + } + let mut result = PathBuf::from(result); + if let Some(segments) = root.path_segments() { + for segment in segments.filter(|s| !s.is_empty()) { + result = result.join(sanitize_segment(segment)); + } + } + + result +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_is_supported_ext() { + assert!(!is_supported_ext(Path::new("tests/subdir/redirects"))); + assert!(!is_supported_ext(Path::new("README.md"))); + assert!(is_supported_ext(Path::new("lib/typescript.d.ts"))); + assert!(is_supported_ext(Path::new("testdata/run/001_hello.js"))); + assert!(is_supported_ext(Path::new("testdata/run/002_hello.ts"))); + assert!(is_supported_ext(Path::new("foo.jsx"))); + assert!(is_supported_ext(Path::new("foo.tsx"))); + assert!(is_supported_ext(Path::new("foo.TS"))); + assert!(is_supported_ext(Path::new("foo.TSX"))); + assert!(is_supported_ext(Path::new("foo.JS"))); + assert!(is_supported_ext(Path::new("foo.JSX"))); + assert!(is_supported_ext(Path::new("foo.mjs"))); + assert!(is_supported_ext(Path::new("foo.mts"))); + assert!(is_supported_ext(Path::new("foo.cjs"))); + assert!(is_supported_ext(Path::new("foo.cts"))); + assert!(!is_supported_ext(Path::new("foo.mjsx"))); + } + + #[test] + fn test_specifier_to_file_path() { + run_success_test("file:///", "/"); + run_success_test("file:///test", "/test"); + run_success_test("file:///dir/test/test.txt", "/dir/test/test.txt"); + run_success_test( + "file:///dir/test%20test/test.txt", + "/dir/test test/test.txt", + ); + + fn run_success_test(specifier: &str, expected_path: &str) { + let result = + specifier_to_file_path(&ModuleSpecifier::parse(specifier).unwrap()) + .unwrap(); + assert_eq!(result, PathBuf::from(expected_path)); + } + } + + #[test] + fn test_ensure_directory_specifier() { + run_test("file:///", "file:///"); + run_test("file:///test", "file:///test/"); + run_test("file:///test/", "file:///test/"); + run_test("file:///test/other", "file:///test/other/"); + run_test("file:///test/other/", "file:///test/other/"); + + fn run_test(specifier: &str, expected: &str) { + let result = + ensure_directory_specifier(ModuleSpecifier::parse(specifier).unwrap()); + assert_eq!(result.to_string(), expected); + } + } + + #[test] + fn test_specifier_parent() { + run_test("file:///", "file:///"); + run_test("file:///test", "file:///"); + run_test("file:///test/", "file:///"); + run_test("file:///test/other", "file:///test/"); + run_test("file:///test/other.txt", "file:///test/"); + run_test("file:///test/other/", "file:///test/"); + + fn run_test(specifier: &str, expected: &str) { + let result = + specifier_parent(&ModuleSpecifier::parse(specifier).unwrap()); + assert_eq!(result.to_string(), expected); + } + } + + #[test] + fn test_relative_specifier() { + let fixtures: Vec<(&str, &str, Option<&str>)> = vec![ + ("file:///from", "file:///to", Some("./to")), + ("file:///from", "file:///from/other", Some("./from/other")), + ("file:///from", "file:///from/other/", Some("./from/other/")), + ("file:///from", "file:///other/from", Some("./other/from")), + ("file:///from/", "file:///other/from", Some("../other/from")), + ("file:///from", "file:///other/from/", Some("./other/from/")), + ( + "file:///from", + "file:///to/other.txt", + Some("./to/other.txt"), + ), + ( + "file:///from/test", + "file:///to/other.txt", + Some("../to/other.txt"), + ), + ( + "file:///from/other.txt", + "file:///to/other.txt", + Some("../to/other.txt"), + ), + ( + "https://deno.land/x/a/b/d.ts", + "https://deno.land/x/a/b/c.ts", + Some("./c.ts"), + ), + ( + "https://deno.land/x/a/b/d.ts", + "https://deno.land/x/a/c.ts", + Some("../c.ts"), + ), + ( + "https://deno.land/x/a/b/d.ts", + "https://deno.land/x/a/b/c/d.ts", + Some("./c/d.ts"), + ), + ( + "https://deno.land/x/a/b/c/", + "https://deno.land/x/a/b/c/d.ts", + Some("./d.ts"), + ), + ( + "https://deno.land/x/a/b/c/", + "https://deno.land/x/a/b/c/d/e.ts", + Some("./d/e.ts"), + ), + ( + "https://deno.land/x/a/b/c/f.ts", + "https://deno.land/x/a/b/c/d/e.ts", + Some("./d/e.ts"), + ), + ( + "https://deno.land/x/a/b/d.ts", + "https://deno.land/x/a/c.ts?foo=bar", + Some("../c.ts?foo=bar"), + ), + ( + "https://deno.land/x/a/b/d.ts?foo=bar", + "https://deno.land/x/a/b/c.ts", + Some("./c.ts"), + ), + ("file:///a/b/d.ts", "file:///a/b/c.ts", Some("./c.ts")), + ("https://deno.land/x/a/b/c.ts", "file:///a/b/c.ts", None), + ( + "https://deno.land/", + "https://deno.land/x/a/b/c.ts", + Some("./x/a/b/c.ts"), + ), + ( + "https://deno.land/x/d/e/f.ts", + "https://deno.land/x/a/b/c.ts", + Some("../../a/b/c.ts"), + ), + ]; + for (from_str, to_str, expected) in fixtures { + let from = ModuleSpecifier::parse(from_str).unwrap(); + let to = ModuleSpecifier::parse(to_str).unwrap(); + let actual = relative_specifier(&from, &to); + assert_eq!( + actual.as_deref(), + expected, + "from: \"{}\" to: \"{}\"", + from_str, + to_str + ); + } + } + + #[test] + fn test_path_has_trailing_slash() { + #[cfg(not(windows))] + { + run_test("/Users/johndoe/Desktop/deno-project/target/", true); + run_test(r"/Users/johndoe/deno-project/target//", true); + run_test("/Users/johndoe/Desktop/deno-project", false); + run_test(r"/Users/johndoe/deno-project\", false); + } + + #[cfg(windows)] + { + run_test(r"C:\test\deno-project\", true); + run_test(r"C:\test\deno-project\\", true); + run_test(r"C:\test\file.txt", false); + run_test(r"C:\test\file.txt/", false); + } + + fn run_test(path_str: &str, expected: bool) { + let path = Path::new(path_str); + let result = path_has_trailing_slash(path); + assert_eq!(result, expected); + } + } + + #[test] + fn test_path_with_stem_suffix() { + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/"), "_2"), + PathBuf::from("/_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test"), "_2"), + PathBuf::from("/test_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.txt"), "_2"), + PathBuf::from("/test_2.txt") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test/subdir"), "_2"), + PathBuf::from("/test/subdir_2") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test/subdir.other.txt"), "_2"), + PathBuf::from("/test/subdir.other_2.txt") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.ts"), "_2"), + PathBuf::from("/test_2.d.ts") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.D.TS"), "_2"), + PathBuf::from("/test_2.D.TS") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.mts"), "_2"), + PathBuf::from("/test_2.d.mts") + ); + assert_eq!( + path_with_stem_suffix(&PathBuf::from("/test.d.cts"), "_2"), + PathBuf::from("/test_2.d.cts") + ); + } +} diff --git a/cli/progress_bar.rs b/cli/util/progress_bar.rs similarity index 100% rename from cli/progress_bar.rs rename to cli/util/progress_bar.rs diff --git a/cli/text_encoding.rs b/cli/util/text_encoding.rs similarity index 100% rename from cli/text_encoding.rs rename to cli/util/text_encoding.rs diff --git a/cli/unix_util.rs b/cli/util/unix.rs similarity index 100% rename from cli/unix_util.rs rename to cli/util/unix.rs diff --git a/cli/windows_util.rs b/cli/util/windows.rs similarity index 100% rename from cli/windows_util.rs rename to cli/util/windows.rs diff --git a/cli/worker.rs b/cli/worker.rs index ad7b4e8ed2..4762bab1f2 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -24,7 +24,6 @@ use deno_runtime::worker::WorkerOptions; use deno_runtime::BootstrapOptions; use crate::args::DenoSubcommand; -use crate::checksum; use crate::errors; use crate::module_loader::CliModuleLoader; use crate::node; @@ -34,6 +33,7 @@ use crate::proc_state::ProcState; use crate::tools; use crate::tools::coverage::CoverageCollector; use crate::tools::test::TestMode; +use crate::util::checksum; use crate::version; pub struct CliMainWorker { diff --git a/tools/napi/generate_symbols_lists.js b/tools/napi/generate_symbols_lists.js index fbef69edde..2da9ccff79 100755 --- a/tools/napi/generate_symbols_lists.js +++ b/tools/napi/generate_symbols_lists.js @@ -1,7 +1,7 @@ #!/usr/bin/env -S deno run --unstable --allow-read --allow-write // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -import exports from "../../cli/napi_sym/symbol_exports.json" assert { +import exports from "../../cli/napi/sym/symbol_exports.json" assert { type: "json", };