foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-24 08:09:08 -05:00
Code Issues Packages 1 Wiki Activity

refactor: move FileCollector to deno_config (#24433)

Browse source
This commit is contained in:
David Sherret 2024-07-05 17:53:09 -04:00 committed by GitHub
parent d4d3a3c54f
commit 80df9aec1d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 117 additions and 635 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Cargo.lock generated
View file

@ -1148,7 +1148,6 @@ dependencies = [
"fs3",
"glibc_version",
"glob",
"ignore",
"import_map",
"indexmap",
"jsonc-parser",
@ -1308,13 +1307,14 @@ dependencies = [
[[package]]
name = "deno_config"
version = "0.19.1"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddc80f97cffe52c9a430201f288111fc89d33491b1675c0e01feb3a497ce76b3"
checksum = "64772162a8e8c1b3a9c48b4a0924e29f5b8f0ae23ea2027361937e96d04d493d"
dependencies = [
"anyhow",
"deno_semver",
"glob",
"ignore",
"import_map",
"indexmap",
"jsonc-parser",

2
Cargo.toml
View file

@ -101,7 +101,7 @@ console_static_text = "=0.8.1"
data-encoding = "2.3.3"
data-url = "=0.3.0"
deno_cache_dir = "=0.10.0"
deno_config = { version = "=0.19.1", default-features = false }
deno_config = { version = "=0.20.0", default-features = false }
dlopen2 = "0.6.1"
ecb = "=0.1.2"
elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] }

3
cli/Cargo.toml
View file

@ -107,7 +107,6 @@ faster-hex.workspace = true
flate2.workspace = true
fs3.workspace = true
glob = "0.3.1"
ignore = "0.4"
import_map = { version = "=0.20.0", features = ["ext"] }
indexmap.workspace = true
jsonc-parser.workspace = true
@ -149,7 +148,6 @@ tower-lsp.workspace = true
twox-hash.workspace = true
typed-arena = "=2.0.1"
uuid = { workspace = true, features = ["serde"] }
walkdir = "=2.3.2"
zeromq.workspace = true
zstd.workspace = true
@ -164,6 +162,7 @@ nix.workspace = true
deno_bench_util.workspace = true
pretty_assertions.workspace = true
test_util.workspace = true
walkdir = "=2.3.2"
[package.metadata.winres]
# This section defines the metadata that appears in the deno.exe PE header.

68
cli/args/mod.rs
View file

@ -10,9 +10,11 @@ mod package_json;
use deno_ast::SourceMapOption;
use deno_config::workspace::CreateResolverOptions;
use deno_config::workspace::PackageJsonDepResolution;
use deno_config::workspace::VendorEnablement;
use deno_config::workspace::Workspace;
use deno_config::workspace::WorkspaceDiscoverOptions;
use deno_config::workspace::WorkspaceDiscoverStart;
use deno_config::workspace::WorkspaceEmptyOptions;
use deno_config::workspace::WorkspaceMemberContext;
use deno_config::workspace::WorkspaceResolver;
use deno_config::WorkspaceLintConfig;
@ -778,7 +780,6 @@ pub struct CliOptions {
flags: Flags,
initial_cwd: PathBuf,
maybe_node_modules_folder: Option<PathBuf>,
maybe_vendor_folder: Option<PathBuf>,
npmrc: Arc<ResolvedNpmRc>,
maybe_lockfile: Option<Arc<CliLockfile>>,
overrides: CliOptionOverrides,
@ -822,15 +823,6 @@ impl CliOptions {
root_folder.pkg_json.as_deref(),
)
.with_context(|| "Resolving node_modules folder.")?;
let maybe_vendor_folder = if force_global_cache {
None
} else {
resolve_vendor_folder(
&initial_cwd,
&flags,
root_folder.deno_json.as_deref(),
)
};
if let Some(env_file_name) = &flags.env_file {
match from_filename(env_file_name) {
@ -859,7 +851,6 @@ impl CliOptions {
maybe_lockfile,
npmrc,
maybe_node_modules_folder,
maybe_vendor_folder,
overrides: Default::default(),
workspace,
disable_deprecated_api_warning,
@ -871,6 +862,10 @@ impl CliOptions {
let initial_cwd =
std::env::current_dir().with_context(|| "Failed getting cwd.")?;
let config_fs_adapter = DenoConfigFsAdapter::new(&RealFs);
let maybe_vendor_override = flags.vendor.map(|v| match v {
true => VendorEnablement::Enable { cwd: &initial_cwd },
false => VendorEnablement::Disable,
});
let resolve_workspace_discover_options = || {
let additional_config_file_names: &'static [&'static str] =
if matches!(flags.subcommand, DenoSubcommand::Publish(..)) {
@ -899,8 +894,16 @@ impl CliOptions {
config_parse_options,
additional_config_file_names,
discover_pkg_json,
maybe_vendor_override,
}
};
let resolve_empty_options = || WorkspaceEmptyOptions {
root_dir: Arc::new(
ModuleSpecifier::from_directory_path(&initial_cwd).unwrap(),
),
use_vendor_dir: maybe_vendor_override
.unwrap_or(VendorEnablement::Disable),
};
let workspace = match &flags.config_flag {
deno_config::ConfigFlag::Discover => {
@ -910,9 +913,7 @@ impl CliOptions {
&resolve_workspace_discover_options(),
)?
} else {
Workspace::empty(Arc::new(
ModuleSpecifier::from_directory_path(&initial_cwd).unwrap(),
))
Workspace::empty(resolve_empty_options())
}
}
deno_config::ConfigFlag::Path(path) => {
@ -922,9 +923,9 @@ impl CliOptions {
&resolve_workspace_discover_options(),
)?
}
deno_config::ConfigFlag::Disabled => Workspace::empty(Arc::new(
ModuleSpecifier::from_directory_path(&initial_cwd).unwrap(),
)),
deno_config::ConfigFlag::Disabled => {
Workspace::empty(resolve_empty_options())
}
};
for diagnostic in workspace.diagnostics() {
@ -1258,7 +1259,6 @@ impl CliOptions {
flags: self.flags.clone(),
initial_cwd: self.initial_cwd.clone(),
maybe_node_modules_folder: Some(path),
maybe_vendor_folder: self.maybe_vendor_folder.clone(),
npmrc: self.npmrc.clone(),
maybe_lockfile: self.maybe_lockfile.clone(),
workspace: self.workspace.clone(),
@ -1276,7 +1276,7 @@ impl CliOptions {
}
pub fn vendor_dir_path(&self) -> Option<&PathBuf> {
self.maybe_vendor_folder.as_ref()
self.workspace.vendor_dir_path()
}
pub fn resolve_root_cert_store_provider(
@ -1801,31 +1801,6 @@ fn resolve_node_modules_folder(
Ok(Some(canonicalize_path_maybe_not_exists(&path)?))
}
fn resolve_vendor_folder(
cwd: &Path,
flags: &Flags,
maybe_config_file: Option<&ConfigFile>,
) -> Option<PathBuf> {
let use_vendor_dir = flags
.vendor
.or_else(|| maybe_config_file.and_then(|c| c.json.vendor))
.unwrap_or(false);
// Unlike the node_modules directory, there is no need to canonicalize
// this directory because it's just used as a cache and the resolved
// specifier is not based on the canonicalized path (unlike the modules
// in the node_modules folder).
if !use_vendor_dir {
None
} else if let Some(config_path) = maybe_config_file
.as_ref()
.and_then(|c| c.specifier.to_file_path().ok())
{
Some(config_path.parent().unwrap().join("vendor"))
} else {
Some(cwd.join("vendor"))
}
}
fn resolve_import_map_specifier(
maybe_import_map_path: Option<&str>,
maybe_config_file: Option<&ConfigFile>,
@ -1962,9 +1937,8 @@ pub fn config_to_deno_graph_workspace_member(
#[cfg(test)]
mod test {
use crate::util::fs::FileCollector;
use super::*;
use deno_config::glob::FileCollector;
use pretty_assertions::assert_eq;
#[test]
@ -2109,7 +2083,7 @@ mod test {
let mut files = FileCollector::new(|_| true)
.ignore_git_folder()
.ignore_node_modules()
.collect_file_patterns(resolved_files)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, resolved_files)
.unwrap();
files.sort();

8
cli/lsp/config.rs
View file

@ -1299,7 +1299,13 @@ impl ConfigData {
}
};
let vendor_dir = config_file.as_ref().and_then(|c| c.vendor_dir_path());
let vendor_dir = config_file.as_ref().and_then(|c| {
if c.vendor() == Some(true) {
Some(c.specifier.to_file_path().ok()?.parent()?.join("vendor"))
} else {
None
}
});
// Load lockfile
let lockfile = config_file.as_ref().and_then(resolve_lockfile_from_config);

5
cli/lsp/language_server.rs
View file

@ -3568,6 +3568,11 @@ impl Inner {
},
additional_config_file_names: &[],
discover_pkg_json: true,
maybe_vendor_override: if force_global_cache {
Some(deno_config::workspace::VendorEnablement::Disable)
} else {
None
},
},
)?);
let cli_options = CliOptions::new(

2
cli/tools/bench/mod.rs
View file

@ -13,12 +13,12 @@ use crate::tools::test::format_test_error;
use crate::tools::test::TestFilter;
use crate::util::file_watcher;
use crate::util::fs::collect_specifiers;
use crate::util::fs::WalkEntry;
use crate::util::path::is_script_ext;
use crate::util::path::matches_pattern_or_exact_path;
use crate::version::get_user_agent;
use crate::worker::CliMainWorkerFactory;
use deno_config::glob::WalkEntry;
use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::error::JsError;

4
cli/tools/coverage/mod.rs
View file

@ -9,11 +9,11 @@ use crate::factory::CliFactory;
use crate::npm::CliNpmResolver;
use crate::tools::fmt::format_json;
use crate::tools::test::is_supported_test_path;
use crate::util::fs::FileCollector;
use crate::util::text_encoding::source_map_from_code;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_config::glob::PathOrPattern;
use deno_config::glob::PathOrPatternSet;
@ -408,7 +408,7 @@ fn collect_coverages(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(file_patterns)?;
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)?;
let coverage_patterns = FilePatterns {
base: initial_cwd.to_path_buf(),

4
cli/tools/fmt.rs
View file

@ -19,10 +19,10 @@ use crate::factory::CliFactory;
use crate::util::diff::diff;
use crate::util::file_watcher;
use crate::util::fs::canonicalize_path;
use crate::util::fs::FileCollector;
use crate::util::path::get_extension;
use async_trait::async_trait;
use deno_ast::ParsedSource;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::bail;
@ -200,7 +200,7 @@ fn collect_fmt_files(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(files)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
}
/// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks

4
cli/tools/lint/mod.rs
View file

@ -8,6 +8,7 @@ use deno_ast::ModuleSpecifier;
use deno_ast::ParsedSource;
use deno_ast::SourceRange;
use deno_ast::SourceTextInfo;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_config::workspace::Workspace;
use deno_config::workspace::WorkspaceMemberContext;
@ -60,7 +61,6 @@ use crate::tools::fmt::run_parallelized;
use crate::util::file_watcher;
use crate::util::fs::canonicalize_path;
use crate::util::fs::specifier_from_file_path;
use crate::util::fs::FileCollector;
use crate::util::path::is_script_ext;
use crate::util::sync::AtomicFlag;
@ -401,7 +401,7 @@ fn collect_lint_files(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.collect_file_patterns(files)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)
}
#[allow(clippy::print_stdout)]

12
cli/tools/registry/paths.rs
View file

@ -8,12 +8,12 @@ use std::path::PathBuf;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_core::error::AnyError;
use thiserror::Error;
use crate::args::CliOptions;
use crate::util::fs::FileCollector;
use super::diagnostics::PublishDiagnostic;
use super::diagnostics::PublishDiagnosticsCollector;
@ -319,14 +319,14 @@ fn collect_paths(
file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(|e| {
if !e.file_type.is_file() {
if !e.metadata.is_file {
if let Ok(specifier) = ModuleSpecifier::from_file_path(e.path) {
diagnostics_collector.push(PublishDiagnostic::UnsupportedFileType {
specifier,
kind: if e.file_type.is_symlink() {
"symlink".to_owned()
kind: if e.metadata.is_symlink {
"symlink".to_string()
} else {
format!("{:?}", e.file_type)
"Unknown".to_string()
},
});
}
@ -341,5 +341,5 @@ fn collect_paths(
.ignore_node_modules()
.set_vendor_folder(cli_options.vendor_dir_path().map(ToOwned::to_owned))
.use_gitignore()
.collect_file_patterns(file_patterns)
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, file_patterns)
}

2
cli/tools/test/mod.rs
View file

@ -15,7 +15,6 @@ use crate::graph_util::has_graph_root_local_dependent_changed;
use crate::ops;
use crate::util::file_watcher;
use crate::util::fs::collect_specifiers;
use crate::util::fs::WalkEntry;
use crate::util::path::get_extension;
use crate::util::path::is_script_ext;
use crate::util::path::mapped_specifier_for_tsc;
@ -27,6 +26,7 @@ use deno_ast::swc::common::comments::CommentKind;
use deno_ast::MediaType;
use deno_ast::SourceRangedForSpanned;
use deno_config::glob::FilePatterns;
use deno_config::glob::WalkEntry;
use deno_core::anyhow;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context as _;

339
cli/util/fs.rs
View file

@ -1,8 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashSet;
use std::env::current_dir;
use std::fs::FileType;
use std::fs::OpenOptions;
use std::io::Error;
use std::io::ErrorKind;
@ -11,11 +9,12 @@ use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use walkdir::WalkDir;
use deno_config::glob::FileCollector;
use deno_config::glob::FilePatterns;
use deno_config::glob::PathOrPattern;
use deno_config::glob::PathOrPatternSet;
use deno_config::glob::WalkEntry;
use deno_core::anyhow::anyhow;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
@ -25,8 +24,6 @@ use deno_core::ModuleSpecifier;
use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::PathClean;
use crate::util::gitignore::DirGitIgnores;
use crate::util::gitignore::GitIgnoreTree;
use crate::util::path::get_atomic_file_path;
use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle;
@ -270,192 +267,6 @@ pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
Ok(normalize_path(resolved_path))
}
#[derive(Debug, Clone)]
pub struct WalkEntry<'a> {
pub path: &'a Path,
pub file_type: &'a FileType,
pub patterns: &'a FilePatterns,
}
/// Collects file paths that satisfy the given predicate, by recursively walking `files`.
/// If the walker visits a path that is listed in `ignore`, it skips descending into the directory.
pub struct FileCollector<TFilter: Fn(WalkEntry) -> bool> {
file_filter: TFilter,
ignore_git_folder: bool,
ignore_node_modules: bool,
vendor_folder: Option<PathBuf>,
use_gitignore: bool,
}
impl<TFilter: Fn(WalkEntry) -> bool> FileCollector<TFilter> {
pub fn new(file_filter: TFilter) -> Self {
Self {
file_filter,
ignore_git_folder: false,
ignore_node_modules: false,
vendor_folder: None,
use_gitignore: false,
}
}
pub fn ignore_node_modules(mut self) -> Self {
self.ignore_node_modules = true;
self
}
pub fn set_vendor_folder(mut self, vendor_folder: Option<PathBuf>) -> Self {
self.vendor_folder = vendor_folder;
self
}
pub fn ignore_git_folder(mut self) -> Self {
self.ignore_git_folder = true;
self
}
pub fn use_gitignore(mut self) -> Self {
self.use_gitignore = true;
self
}
pub fn collect_file_patterns(
&self,
file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> {
fn is_pattern_matched(
maybe_git_ignore: Option<&DirGitIgnores>,
path: &Path,
is_dir: bool,
file_patterns: &FilePatterns,
) -> bool {
use deno_config::glob::FilePatternsMatch;
let path_kind = match is_dir {
true => deno_config::glob::PathKind::Directory,
false => deno_config::glob::PathKind::File,
};
match file_patterns.matches_path_detail(path, path_kind) {
FilePatternsMatch::Passed => {
// check gitignore
let is_gitignored = maybe_git_ignore
.as_ref()
.map(|git_ignore| git_ignore.is_ignored(path, is_dir))
.unwrap_or(false);
!is_gitignored
}
FilePatternsMatch::PassedOptedOutExclude => true,
FilePatternsMatch::Excluded => false,
}
}
let mut maybe_git_ignores = if self.use_gitignore {
// Override explicitly specified include paths in the
// .gitignore file. This does not apply to globs because
// that is way too complicated to reason about.
let include_paths = file_patterns
.include
.as_ref()
.map(|include| {
include
.inner()
.iter()
.filter_map(|path_or_pattern| {
if let PathOrPattern::Path(p) = path_or_pattern {
Some(p.clone())
} else {
None
}
})
.collect::<Vec<_>>()
})
.unwrap_or_default();
Some(GitIgnoreTree::new(
Arc::new(deno_runtime::deno_fs::RealFs),
include_paths,
))
} else {
None
};
let mut target_files = Vec::new();
let mut visited_paths = HashSet::new();
let file_patterns_by_base = file_patterns.split_by_base();
for file_patterns in file_patterns_by_base {
let file = normalize_path(&file_patterns.base);
// use an iterator in order to minimize the number of file system operations
let mut iterator = WalkDir::new(&file)
.follow_links(false) // the default, but be explicit
.into_iter();
loop {
let e = match iterator.next() {
None => break,
Some(Err(_)) => continue,
Some(Ok(entry)) => entry,
};
let file_type = e.file_type();
let is_dir = file_type.is_dir();
let path = e.path().to_path_buf();
let maybe_gitignore =
maybe_git_ignores.as_mut().and_then(|git_ignores| {
if is_dir {
git_ignores.get_resolved_git_ignore_for_dir(&path)
} else {
git_ignores.get_resolved_git_ignore_for_file(&path)
}
});
if !is_pattern_matched(
maybe_gitignore.as_deref(),
&path,
is_dir,
&file_patterns,
) {
if is_dir {
iterator.skip_current_dir();
}
} else if is_dir {
// allow the user to opt out of ignoring by explicitly specifying the dir
let opt_out_ignore = file == path;
let should_ignore_dir = !opt_out_ignore && self.is_ignored_dir(&path);
if should_ignore_dir || !visited_paths.insert(path.clone()) {
iterator.skip_current_dir();
}
} else if (self.file_filter)(WalkEntry {
path: &path,
file_type: &file_type,
patterns: &file_patterns,
}) && visited_paths.insert(path.clone())
{
target_files.push(path);
}
}
}
Ok(target_files)
}
fn is_ignored_dir(&self, path: &Path) -> bool {
path
.file_name()
.map(|dir_name| {
let dir_name = dir_name.to_string_lossy().to_lowercase();
let is_ignored_file = match dir_name.as_str() {
"node_modules" => self.ignore_node_modules,
".git" => self.ignore_git_folder,
_ => false,
};
is_ignored_file
})
.unwrap_or(false)
|| self.is_vendor_folder(path)
}
fn is_vendor_folder(&self, path: &Path) -> bool {
self
.vendor_folder
.as_ref()
.map(|vendor_folder| path == *vendor_folder)
.unwrap_or(false)
}
}
/// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`.
/// Specifiers that start with http and https are left intact.
/// Note: This ignores all .git and node_modules folders.
@ -501,7 +312,7 @@ pub fn collect_specifiers(
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(vendor_folder)
.collect_file_patterns(files)?;
.collect_file_patterns(&deno_config::fs::RealDenoConfigFs, files)?;
let mut collected_files_as_urls = collected_files
.iter()
.map(|f| specifier_from_file_path(f).unwrap())
@ -953,150 +764,6 @@ mod tests {
assert_eq!(resolve_from_cwd(expected).unwrap(), absolute_expected);
}
#[test]
fn test_collect_files() {
fn create_files(dir_path: &PathRef, files: &[&str]) {
dir_path.create_dir_all();
for f in files {
dir_path.join(f).write("");
}
}
// dir.ts
// ├── a.ts
// ├── b.js
// ├── child
// | ├── git
// | | └── git.js
// | ├── node_modules
// | | └── node_modules.js
// | ├── vendor
// | | └── vendor.js
// │ ├── e.mjs
// │ ├── f.mjsx
// │ ├── .foo.TS
// │ └── README.md
// ├── c.tsx
// ├── d.jsx
// └── ignore
// ├── g.d.ts
// └── .gitignore
let t = TempDir::new();
let root_dir_path = t.path().join("dir.ts");
let root_dir_files = ["a.ts", "b.js", "c.tsx", "d.jsx"];
create_files(&root_dir_path, &root_dir_files);
let child_dir_path = root_dir_path.join("child");
let child_dir_files = ["e.mjs", "f.mjsx", ".foo.TS", "README.md"];
create_files(&child_dir_path, &child_dir_files);
t.create_dir_all("dir.ts/child/node_modules");
t.write("dir.ts/child/node_modules/node_modules.js", "");
t.create_dir_all("dir.ts/child/.git");
t.write("dir.ts/child/.git/git.js", "");
t.create_dir_all("dir.ts/child/vendor");
t.write("dir.ts/child/vendor/vendor.js", "");
let ignore_dir_path = root_dir_path.join("ignore");
let ignore_dir_files = ["g.d.ts", ".gitignore"];
create_files(&ignore_dir_path, &ignore_dir_files);
let file_patterns = FilePatterns {
base: root_dir_path.to_path_buf(),
include: None,
exclude: PathOrPatternSet::new(vec![PathOrPattern::Path(
ignore_dir_path.to_path_buf(),
)]),
};
let file_collector = FileCollector::new(|e| {
// exclude dotfiles
e.path
.file_name()
.and_then(|f| f.to_str())
.map(|f| !f.starts_with('.'))
.unwrap_or(false)
});
let result = file_collector
.collect_file_patterns(file_patterns.clone())
.unwrap();
let expected = [
"README.md",
"a.ts",
"b.js",
"c.tsx",
"d.jsx",
"e.mjs",
"f.mjsx",
"git.js",
"node_modules.js",
"vendor.js",
];
let mut file_names = result
.into_iter()
.map(|r| r.file_name().unwrap().to_string_lossy().to_string())
.collect::<Vec<_>>();
file_names.sort();
assert_eq!(file_names, expected);
// test ignoring the .git and node_modules folder
let file_collector = file_collector
.ignore_git_folder()
.ignore_node_modules()
.set_vendor_folder(Some(child_dir_path.join("vendor").to_path_buf()));
let result = file_collector
.collect_file_patterns(file_patterns.clone())
.unwrap();
let expected = [
"README.md",
"a.ts",
"b.js",
"c.tsx",
"d.jsx",
"e.mjs",
"f.mjsx",
];
let mut file_names = result
.into_iter()
.map(|r| r.file_name().unwrap().to_string_lossy().to_string())
.collect::<Vec<_>>();
file_names.sort();
assert_eq!(file_names, expected);
// test opting out of ignoring by specifying the dir
let file_patterns = FilePatterns {
base: root_dir_path.to_path_buf(),
include: Some(PathOrPatternSet::new(vec![
PathOrPattern::Path(root_dir_path.to_path_buf()),
PathOrPattern::Path(
root_dir_path.to_path_buf().join("child/node_modules/"),
),
])),
exclude: PathOrPatternSet::new(vec![PathOrPattern::Path(
ignore_dir_path.to_path_buf(),
)]),
};
let result = file_collector.collect_file_patterns(file_patterns).unwrap();
let expected = [
"README.md",
"a.ts",
"b.js",
"c.tsx",
"d.jsx",
"e.mjs",
"f.mjsx",
"node_modules.js",
];
let mut file_names = result
.into_iter()
.map(|r| r.file_name().unwrap().to_string_lossy().to_string())
.collect::<Vec<_>>();
file_names.sort();
assert_eq!(file_names, expected);
}
#[test]
fn test_collect_specifiers() {
fn create_files(dir_path: &PathRef, files: &[&str]) {

178
cli/util/gitignore.rs
View file

@ -1,178 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
use std::rc::Rc;
use std::sync::Arc;
/// Resolved gitignore for a directory.
pub struct DirGitIgnores {
current: Option<Rc<ignore::gitignore::Gitignore>>,
parent: Option<Rc<DirGitIgnores>>,
}
impl DirGitIgnores {
pub fn is_ignored(&self, path: &Path, is_dir: bool) -> bool {
let mut is_ignored = false;
if let Some(parent) = &self.parent {
is_ignored = parent.is_ignored(path, is_dir);
}
if let Some(current) = &self.current {
match current.matched(path, is_dir) {
ignore::Match::None => {}
ignore::Match::Ignore(_) => {
is_ignored = true;
}
ignore::Match::Whitelist(_) => {
is_ignored = false;
}
}
}
is_ignored
}
}
/// Resolves gitignores in a directory tree taking into account
/// ancestor gitignores that may be found in a directory.
pub struct GitIgnoreTree {
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
ignores: HashMap<PathBuf, Option<Rc<DirGitIgnores>>>,
include_paths: Vec<PathBuf>,
}
impl GitIgnoreTree {
pub fn new(
fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
// paths that should override what's in the gitignore
include_paths: Vec<PathBuf>,
) -> Self {
Self {
fs,
ignores: Default::default(),
include_paths,
}
}
pub fn get_resolved_git_ignore_for_dir(
&mut self,
dir_path: &Path,
) -> Option<Rc<DirGitIgnores>> {
// for directories, provide itself in order to tell
// if it should stop searching for gitignores because
// maybe this dir_path is a .git directory
let parent = dir_path.parent()?;
self.get_resolved_git_ignore_inner(parent, Some(dir_path))
}
pub fn get_resolved_git_ignore_for_file(
&mut self,
file_path: &Path,
) -> Option<Rc<DirGitIgnores>> {
let dir_path = file_path.parent()?;
self.get_resolved_git_ignore_inner(dir_path, None)
}
fn get_resolved_git_ignore_inner(
&mut self,
dir_path: &Path,
maybe_parent: Option<&Path>,
) -> Option<Rc<DirGitIgnores>> {
let maybe_resolved = self.ignores.get(dir_path).cloned();
if let Some(resolved) = maybe_resolved {
resolved
} else {
let resolved = self.resolve_gitignore_in_dir(dir_path, maybe_parent);
self.ignores.insert(dir_path.to_owned(), resolved.clone());
resolved
}
}
fn resolve_gitignore_in_dir(
&mut self,
dir_path: &Path,
maybe_parent: Option<&Path>,
) -> Option<Rc<DirGitIgnores>> {
if let Some(parent) = maybe_parent {
// stop searching if the parent dir had a .git directory in it
if self.fs.exists_sync(&parent.join(".git")) {
return None;
}
}
let parent = dir_path.parent().and_then(|parent| {
self.get_resolved_git_ignore_inner(parent, Some(dir_path))
});
let current = self
.fs
.read_text_file_lossy_sync(&dir_path.join(".gitignore"), None)
.ok()
.and_then(|text| {
let mut builder = ignore::gitignore::GitignoreBuilder::new(dir_path);
for line in text.lines() {
builder.add_line(None, line).ok()?;
}
// override the gitignore contents to include these paths
for path in &self.include_paths {
if let Ok(suffix) = path.strip_prefix(dir_path) {
let suffix = suffix.to_string_lossy().replace('\\', "/");
let _ignore = builder.add_line(None, &format!("!/{}", suffix));
if !suffix.ends_with('/') {
let _ignore = builder.add_line(None, &format!("!/{}/", suffix));
}
}
}
let gitignore = builder.build().ok()?;
Some(Rc::new(gitignore))
});
if parent.is_none() && current.is_none() {
None
} else {
Some(Rc::new(DirGitIgnores { current, parent }))
}
}
}
#[cfg(test)]
mod test {
use deno_runtime::deno_fs::InMemoryFs;
use super::*;
#[test]
fn git_ignore_tree() {
let fs = InMemoryFs::default();
fs.setup_text_files(vec![
("/.gitignore".into(), "file.txt".into()),
("/sub_dir/.gitignore".into(), "data.txt".into()),
(
"/sub_dir/sub_dir/.gitignore".into(),
"!file.txt\nignore.txt".into(),
),
]);
let mut ignore_tree = GitIgnoreTree::new(Arc::new(fs), Vec::new());
let mut run_test = |path: &str, expected: bool| {
let path = PathBuf::from(path);
let gitignore =
ignore_tree.get_resolved_git_ignore_for_file(&path).unwrap();
assert_eq!(
gitignore.is_ignored(&path, /* is_dir */ false),
expected,
"Path: {}",
path.display()
);
};
run_test("/file.txt", true);
run_test("/other.txt", false);
run_test("/data.txt", false);
run_test("/sub_dir/file.txt", true);
run_test("/sub_dir/other.txt", false);
run_test("/sub_dir/data.txt", true);
run_test("/sub_dir/sub_dir/file.txt", false); // unignored up here
run_test("/sub_dir/sub_dir/sub_dir/file.txt", false);
run_test("/sub_dir/sub_dir/sub_dir/ignore.txt", true);
run_test("/sub_dir/sub_dir/ignore.txt", true);
run_test("/sub_dir/ignore.txt", false);
run_test("/ignore.txt", false);
}
}

1
cli/util/mod.rs
View file

@ -9,7 +9,6 @@ pub mod display;
pub mod draw_thread;
pub mod file_watcher;
pub mod fs;
pub mod gitignore;
pub mod logger;
pub mod path;
pub mod progress_bar;

35
cli/util/sync/atomic_flag.rs
View file

@ -1,35 +0,0 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::sync::atomic::AtomicBool;
use std::sync::atomic::Ordering;
/// Simplifies the use of an atomic boolean as a flag.
#[derive(Debug, Default)]
pub struct AtomicFlag(AtomicBool);
impl AtomicFlag {
/// Raises the flag returning if the raise was successful.
pub fn raise(&self) -> bool {
!self.0.swap(true, Ordering::SeqCst)
}
/// Gets if the flag is raised.
pub fn is_raised(&self) -> bool {
self.0.load(Ordering::SeqCst)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn atomic_flag_raises() {
let flag = AtomicFlag::default();
assert!(!flag.is_raised()); // false by default
assert!(flag.raise());
assert!(flag.is_raised());
assert!(!flag.raise());
assert!(flag.is_raised());
}
}

4
cli/util/sync/mod.rs
View file

@ -1,14 +1,14 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
mod async_flag;
mod atomic_flag;
mod sync_read_async_write_lock;
mod task_queue;
mod value_creator;
pub use async_flag::AsyncFlag;
pub use atomic_flag::AtomicFlag;
pub use sync_read_async_write_lock::SyncReadAsyncWriteLock;
pub use task_queue::TaskQueue;
pub use task_queue::TaskQueuePermit;
pub use value_creator::MultiRuntimeAsyncValueCreator;
// todo(dsherret): this being in the unsync module is slightly confusing, but it's Sync
pub use deno_core::unsync::AtomicFlag;

71
ext/fs/interface.rs
View file

@ -343,23 +343,68 @@ impl<'a> DenoConfigFsAdapter<'a> {
}
impl<'a> deno_config::fs::DenoConfigFs for DenoConfigFsAdapter<'a> {
fn read_to_string(&self, path: &Path) -> Result<String, std::io::Error> {
use deno_io::fs::FsError;
use std::io::ErrorKind;
fn read_to_string_lossy(
&self,
path: &Path,
) -> Result<String, std::io::Error> {
self
.0
.read_text_file_lossy_sync(path, None)
.map_err(|err| match err {
FsError::Io(io) => io,
FsError::FileBusy => std::io::Error::new(ErrorKind::Other, "file busy"),
FsError::NotSupported => {
std::io::Error::new(ErrorKind::Other, "not supported")
}
FsError::PermissionDenied(name) => std::io::Error::new(
ErrorKind::PermissionDenied,
format!("requires {}", name),
),
.map_err(map_deno_fs_to_config_err)
}
fn stat_sync(
&self,
path: &Path,
) -> Result<deno_config::fs::FsMetadata, std::io::Error> {
self
.0
.stat_sync(path)
.map(|stat| deno_config::fs::FsMetadata {
is_file: stat.is_file,
is_directory: stat.is_directory,
is_symlink: stat.is_symlink,
})
.map_err(map_deno_fs_to_config_err)
}
fn read_dir(
&self,
path: &Path,
) -> Result<Vec<deno_config::fs::FsDirEntry>, std::io::Error> {
self
.0
.read_dir_sync(path)
.map_err(map_deno_fs_to_config_err)
.map(|entries| {
entries
.into_iter()
.map(|e| deno_config::fs::FsDirEntry {
path: path.join(e.name),
metadata: deno_config::fs::FsMetadata {
is_file: e.is_file,
is_directory: e.is_directory,
is_symlink: e.is_symlink,
},
})
.collect()
})
}
}
fn map_deno_fs_to_config_err(fs_err: deno_io::fs::FsError) -> std::io::Error {
use deno_io::fs::FsError;
use std::io::ErrorKind;
match fs_err {
FsError::Io(io) => io,
FsError::FileBusy => std::io::Error::new(ErrorKind::Other, "file busy"),
FsError::NotSupported => {
std::io::Error::new(ErrorKind::Other, "not supported")
}
FsError::PermissionDenied(name) => std::io::Error::new(
ErrorKind::PermissionDenied,
format!("requires {}", name),
),
}
}

4
tests/specs/workspaces/non_fatal_diagnostics/lint.out
View file

@ -1,5 +1,5 @@
The 'compilerOptions' field can only be specified in the root workspace deno.json file.
The "compilerOptions" field can only be specified in the root workspace deno.json file.
at file:///[WILDLINE]/sub/deno.json
The 'lint.report' field can only be specified in the root workspace deno.json file.
The "lint.report" field can only be specified in the root workspace deno.json file.
at file:///[WILDLINE]/sub/deno.json
Checked 1 file