foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2025-01-07 06:46:59 -05:00
Code Issues Packages 1 Wiki Activity

perf: skip expanding exclude globs (#21817)

Browse source 
We were calling `expand_glob` on our excludes, which is very expensive
and unnecessary because we can pattern match while traversing instead.

1. Doesn't expand "exclude" globs. Instead pattern matches while walking
the directory.
2. Splits up the "include" into base paths and applicable file patterns.
This causes less pattern matching to occur because we're only pattern
matching on patterns that might match and not ones in completely
unrelated directories.
This commit is contained in:
David Sherret 2024-01-08 12:18:42 -05:00 committed by Bartek Iwańczuk
parent 8a25d21380
commit 972b3e8e65
No known key found for this signature in database
GPG key ID: 0C6BCDDC3B3AD750
15 changed files with 1020 additions and 500 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
cli/args/flags.rs
View file

@ -34,6 +34,35 @@ pub struct FileFlags {
pub include: Vec<PathBuf>, pub include: Vec<PathBuf>,
} }
impl FileFlags {
pub fn with_absolute_paths(self, base: &Path) -> Self {
fn to_absolute_path(path: PathBuf, base: &Path) -> PathBuf {
// todo(dsherret): don't store URLs in PathBufs
if path.starts_with("http:")
|| path.starts_with("https:")
|| path.starts_with("file:")
{
path
} else {
base.join(path)
}
}
Self {
include: self
.include
.into_iter()
.map(|p| to_absolute_path(p, base))
.collect(),
ignore: self
.ignore
.into_iter()
.map(|p| to_absolute_path(p, base))
.collect(),
}
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)] #[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct BenchFlags { pub struct BenchFlags {
pub files: FileFlags, pub files: FileFlags,

136
cli/args/mod.rs
View file

@ -69,7 +69,8 @@ use thiserror::Error;
use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileFetcher;
use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::fs::canonicalize_path_maybe_not_exists;
use crate::util::glob::expand_globs; use crate::util::glob::FilePatterns;
use crate::util::glob::PathOrPatternSet;
use crate::version; use crate::version;
use deno_config::FmtConfig; use deno_config::FmtConfig;
@ -217,7 +218,7 @@ impl CacheSetting {
#[derive(Clone, Debug, Eq, PartialEq)] #[derive(Clone, Debug, Eq, PartialEq)]
pub struct BenchOptions { pub struct BenchOptions {
pub files: FilesConfig, pub files: FilePatterns,
pub filter: Option<String>, pub filter: Option<String>,
pub json: bool, pub json: bool,
pub no_run: bool, pub no_run: bool,
@ -227,12 +228,14 @@ impl BenchOptions {
pub fn resolve( pub fn resolve(
maybe_bench_config: Option<BenchConfig>, maybe_bench_config: Option<BenchConfig>,
maybe_bench_flags: Option<BenchFlags>, maybe_bench_flags: Option<BenchFlags>,
initial_cwd: &Path,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
let bench_flags = maybe_bench_flags.unwrap_or_default(); let bench_flags = maybe_bench_flags.unwrap_or_default();
Ok(Self { Ok(Self {
files: resolve_files( files: resolve_files(
maybe_bench_config.map(|c| c.files), maybe_bench_config.map(|c| c.files),
Some(bench_flags.files), Some(bench_flags.files),
initial_cwd,
)?, )?,
filter: bench_flags.filter, filter: bench_flags.filter,
json: bench_flags.json, json: bench_flags.json,
@ -245,13 +248,14 @@ impl BenchOptions {
pub struct FmtOptions { pub struct FmtOptions {
pub check: bool, pub check: bool,
pub options: FmtOptionsConfig, pub options: FmtOptionsConfig,
pub files: FilesConfig, pub files: FilePatterns,
} }
impl FmtOptions { impl FmtOptions {
pub fn resolve( pub fn resolve(
maybe_fmt_config: Option<FmtConfig>, maybe_fmt_config: Option<FmtConfig>,
maybe_fmt_flags: Option<FmtFlags>, maybe_fmt_flags: Option<FmtFlags>,
initial_cwd: &Path,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
let (maybe_config_options, maybe_config_files) = let (maybe_config_options, maybe_config_files) =
maybe_fmt_config.map(|c| (c.options, c.files)).unzip(); maybe_fmt_config.map(|c| (c.options, c.files)).unzip();
@ -265,6 +269,7 @@ impl FmtOptions {
files: resolve_files( files: resolve_files(
maybe_config_files, maybe_config_files,
maybe_fmt_flags.map(|f| f.files), maybe_fmt_flags.map(|f| f.files),
initial_cwd,
)?, )?,
}) })
} }
@ -311,26 +316,9 @@ fn resolve_fmt_options(
options options
} }
#[derive(Clone, Debug, Default)]
pub struct CheckOptions {
pub exclude: Vec<PathBuf>,
}
impl CheckOptions {
pub fn resolve(
maybe_files_config: Option<FilesConfig>,
) -> Result<Self, AnyError> {
Ok(Self {
exclude: expand_globs(
maybe_files_config.map(|c| c.exclude).unwrap_or_default(),
)?,
})
}
}
#[derive(Clone)] #[derive(Clone)]
pub struct TestOptions { pub struct TestOptions {
pub files: FilesConfig, pub files: FilePatterns,
pub doc: bool, pub doc: bool,
pub no_run: bool, pub no_run: bool,
pub fail_fast: Option<NonZeroUsize>, pub fail_fast: Option<NonZeroUsize>,
@ -347,6 +335,7 @@ impl TestOptions {
pub fn resolve( pub fn resolve(
maybe_test_config: Option<TestConfig>, maybe_test_config: Option<TestConfig>,
maybe_test_flags: Option<TestFlags>, maybe_test_flags: Option<TestFlags>,
initial_cwd: &Path,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
let test_flags = maybe_test_flags.unwrap_or_default(); let test_flags = maybe_test_flags.unwrap_or_default();
@ -354,6 +343,7 @@ impl TestOptions {
files: resolve_files( files: resolve_files(
maybe_test_config.map(|c| c.files), maybe_test_config.map(|c| c.files),
Some(test_flags.files), Some(test_flags.files),
initial_cwd,
)?, )?,
allow_none: test_flags.allow_none, allow_none: test_flags.allow_none,
concurrent_jobs: test_flags concurrent_jobs: test_flags
@ -382,7 +372,7 @@ pub enum LintReporterKind {
#[derive(Clone, Debug, Default)] #[derive(Clone, Debug, Default)]
pub struct LintOptions { pub struct LintOptions {
pub rules: LintRulesConfig, pub rules: LintRulesConfig,
pub files: FilesConfig, pub files: FilePatterns,
pub reporter_kind: LintReporterKind, pub reporter_kind: LintReporterKind,
} }
@ -390,6 +380,7 @@ impl LintOptions {
pub fn resolve( pub fn resolve(
maybe_lint_config: Option<LintConfig>, maybe_lint_config: Option<LintConfig>,
maybe_lint_flags: Option<LintFlags>, maybe_lint_flags: Option<LintFlags>,
initial_cwd: &Path,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
let mut maybe_reporter_kind = let mut maybe_reporter_kind =
maybe_lint_flags.as_ref().and_then(|lint_flags| { maybe_lint_flags.as_ref().and_then(|lint_flags| {
@ -437,7 +428,11 @@ impl LintOptions {
maybe_lint_config.map(|c| (c.files, c.rules)).unzip(); maybe_lint_config.map(|c| (c.files, c.rules)).unzip();
Ok(Self { Ok(Self {
reporter_kind: maybe_reporter_kind.unwrap_or_default(), reporter_kind: maybe_reporter_kind.unwrap_or_default(),
files: resolve_files(maybe_config_files, Some(maybe_file_flags))?, files: resolve_files(
maybe_config_files,
Some(maybe_file_flags),
initial_cwd,
)?,
rules: resolve_lint_rules_options( rules: resolve_lint_rules_options(
maybe_config_rules, maybe_config_rules,
maybe_rules_tags, maybe_rules_tags,
@ -1184,7 +1179,7 @@ impl CliOptions {
} else { } else {
None None
}; };
FmtOptions::resolve(maybe_fmt_config, Some(fmt_flags)) FmtOptions::resolve(maybe_fmt_config, Some(fmt_flags), &self.initial_cwd)
} }
pub fn resolve_lint_options( pub fn resolve_lint_options(
@ -1196,17 +1191,20 @@ impl CliOptions {
} else { } else {
None None
}; };
LintOptions::resolve(maybe_lint_config, Some(lint_flags)) LintOptions::resolve(maybe_lint_config, Some(lint_flags), &self.initial_cwd)
} }
pub fn resolve_check_options(&self) -> Result<CheckOptions, AnyError> { pub fn resolve_config_excludes(&self) -> Result<PathOrPatternSet, AnyError> {
let maybe_files_config = if let Some(config_file) = &self.maybe_config_file let maybe_files_config = if let Some(config_file) = &self.maybe_config_file
{ {
config_file.to_files_config()? config_file.to_files_config()?
} else { } else {
None None
}; };
CheckOptions::resolve(maybe_files_config) PathOrPatternSet::from_absolute_paths(
maybe_files_config.map(|c| c.exclude).unwrap_or_default(),
)
.context("Invalid config file exclude pattern.")
} }
pub fn resolve_test_options( pub fn resolve_test_options(
@ -1218,7 +1216,7 @@ impl CliOptions {
} else { } else {
None None
}; };
TestOptions::resolve(maybe_test_config, Some(test_flags)) TestOptions::resolve(maybe_test_config, Some(test_flags), &self.initial_cwd)
} }
pub fn resolve_bench_options( pub fn resolve_bench_options(
@ -1231,7 +1229,11 @@ impl CliOptions {
} else { } else {
None None
}; };
BenchOptions::resolve(maybe_bench_config, Some(bench_flags)) BenchOptions::resolve(
maybe_bench_config,
Some(bench_flags),
&self.initial_cwd,
)
} }
/// Vector of user script CLI arguments. /// Vector of user script CLI arguments.
@ -1655,24 +1657,29 @@ impl StorageKeyResolver {
fn resolve_files( fn resolve_files(
maybe_files_config: Option<FilesConfig>, maybe_files_config: Option<FilesConfig>,
maybe_file_flags: Option<FileFlags>, maybe_file_flags: Option<FileFlags>,
) -> Result<FilesConfig, AnyError> { initial_cwd: &Path,
let mut result = maybe_files_config.unwrap_or_default(); ) -> Result<FilePatterns, AnyError> {
let mut maybe_files_config = maybe_files_config.unwrap_or_default();
if let Some(file_flags) = maybe_file_flags { if let Some(file_flags) = maybe_file_flags {
let file_flags = file_flags.with_absolute_paths(initial_cwd);
if !file_flags.include.is_empty() { if !file_flags.include.is_empty() {
result.include = Some(file_flags.include); maybe_files_config.include = Some(file_flags.include);
} }
if !file_flags.ignore.is_empty() { if !file_flags.ignore.is_empty() {
result.exclude = file_flags.ignore; maybe_files_config.exclude = file_flags.ignore
} }
} }
// Now expand globs if there are any Ok(FilePatterns {
result.include = match result.include { include: {
Some(include) => Some(expand_globs(include)?), let files = match maybe_files_config.include {
None => None, Some(include) => include,
}; None => vec![initial_cwd.to_path_buf()],
result.exclude = expand_globs(result.exclude)?; };
Some(PathOrPatternSet::from_absolute_paths(files)?)
Ok(result) },
exclude: PathOrPatternSet::from_absolute_paths(maybe_files_config.exclude)
.context("Invalid exclude.")?,
})
} }
/// Resolves the no_prompt value based on the cli flags and environment. /// Resolves the no_prompt value based on the cli flags and environment.
@ -1694,6 +1701,8 @@ pub fn npm_pkg_req_ref_to_binary_command(
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use crate::util::fs::FileCollector;
use super::*; use super::*;
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
@ -1887,6 +1896,7 @@ mod test {
exclude: vec![], exclude: vec![],
}), }),
None, None,
temp_dir_path,
) )
.unwrap_err(); .unwrap_err();
assert!(error.to_string().starts_with("Failed to expand glob")); assert!(error.to_string().starts_with("Failed to expand glob"));
@ -1902,32 +1912,36 @@ mod test {
exclude: vec![temp_dir_path.join("nested/**/*bazz.ts")], exclude: vec![temp_dir_path.join("nested/**/*bazz.ts")],
}), }),
None, None,
temp_dir_path,
) )
.unwrap(); .unwrap();
let mut files = FileCollector::new(|_, _| true)
.ignore_git_folder()
.ignore_node_modules()
.ignore_vendor_folder()
.collect_file_patterns(resolved_files)
.unwrap();
files.sort();
assert_eq!( assert_eq!(
resolved_files.include, files,
Some(vec![
temp_dir_path.join("data/test1.js"),
temp_dir_path.join("data/test1.ts"),
temp_dir_path.join("nested/foo/bar.ts"),
temp_dir_path.join("nested/foo/bazz.ts"),
temp_dir_path.join("nested/foo/fizz.ts"),
temp_dir_path.join("nested/foo/foo.ts"),
temp_dir_path.join("nested/fizz/bar.ts"),
temp_dir_path.join("nested/fizz/bazz.ts"),
temp_dir_path.join("nested/fizz/fizz.ts"),
temp_dir_path.join("nested/fizz/foo.ts"),
temp_dir_path.join("pages/[id].ts"),
])
);
assert_eq!(
resolved_files.exclude,
vec![ vec![
temp_dir_path.join("nested/fizz/bazz.ts"), "data/test1.js",
temp_dir_path.join("nested/foo/bazz.ts"), "data/test1.ts",
"nested/fizz/bar.ts",
"nested/fizz/fizz.ts",
"nested/fizz/foo.ts",
"nested/foo/bar.ts",
"nested/foo/fizz.ts",
"nested/foo/foo.ts",
"pages/[id].ts",
] ]
) .into_iter()
.map(|p| normalize_path(temp_dir_path.join(p)))
.collect::<Vec<_>>()
);
} }
#[test] #[test]

14
cli/graph_util.rs
View file

@ -16,6 +16,8 @@ use crate::resolver::SloppyImportsResolver;
use crate::tools::check; use crate::tools::check;
use crate::tools::check::TypeChecker; use crate::tools::check::TypeChecker;
use crate::util::file_watcher::WatcherCommunicator; use crate::util::file_watcher::WatcherCommunicator;
use crate::util::fs::canonicalize_path;
use crate::util::path::specifier_to_file_path;
use crate::util::sync::TaskQueue; use crate::util::sync::TaskQueue;
use crate::util::sync::TaskQueuePermit; use crate::util::sync::TaskQueuePermit;
@ -677,7 +679,7 @@ impl ModuleGraphContainer {
pub fn has_graph_root_local_dependent_changed( pub fn has_graph_root_local_dependent_changed(
graph: &ModuleGraph, graph: &ModuleGraph,
root: &ModuleSpecifier, root: &ModuleSpecifier,
changed_specifiers: &HashSet<ModuleSpecifier>, canonicalized_changed_paths: &HashSet<PathBuf>,
) -> bool { ) -> bool {
let roots = vec![root.clone()]; let roots = vec![root.clone()];
let mut dependent_specifiers = graph.walk( let mut dependent_specifiers = graph.walk(
@ -689,11 +691,15 @@ pub fn has_graph_root_local_dependent_changed(
}, },
); );
while let Some((s, _)) = dependent_specifiers.next() { while let Some((s, _)) = dependent_specifiers.next() {
if s.scheme() != "file" { if let Ok(path) = specifier_to_file_path(s) {
if let Ok(path) = canonicalize_path(&path) {
if canonicalized_changed_paths.contains(&path) {
return true;
}
}
} else {
// skip walking this remote module's dependencies // skip walking this remote module's dependencies
dependent_specifiers.skip_previous_dependencies(); dependent_specifiers.skip_previous_dependencies();
} else if changed_specifiers.contains(s) {
return true;
} }
} }
false false

202
cli/lsp/documents.rs
View file

@ -24,6 +24,7 @@ use crate::resolver::SloppyImportsFsEntry;
use crate::resolver::SloppyImportsResolution; use crate::resolver::SloppyImportsResolution;
use crate::resolver::SloppyImportsResolver; use crate::resolver::SloppyImportsResolver;
use crate::util::glob; use crate::util::glob;
use crate::util::glob::FilePatterns;
use crate::util::path::specifier_to_file_path; use crate::util::path::specifier_to_file_path;
use crate::util::text_encoding; use crate::util::text_encoding;
@ -62,6 +63,7 @@ use std::fs::ReadDir;
use std::ops::Range; use std::ops::Range;
use std::path::Path; use std::path::Path;
use std::path::PathBuf; use std::path::PathBuf;
use std::rc::Rc;
use std::str::FromStr; use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use tower_lsp::lsp_types as lsp; use tower_lsp::lsp_types as lsp;
@ -1852,13 +1854,14 @@ fn analyze_module(
} }
} }
#[derive(Debug)]
enum PendingEntry { enum PendingEntry {
/// File specified as a root url. /// File specified as a root url.
SpecifiedRootFile(PathBuf), SpecifiedRootFile(PathBuf),
/// Directory that is queued to read. /// Directory that is queued to read.
Dir(PathBuf), Dir(PathBuf, Rc<FilePatterns>),
/// The current directory being read. /// The current directory being read.
ReadDir(Box<ReadDir>), ReadDir(Box<ReadDir>, Rc<FilePatterns>),
} }
struct PreloadDocumentFinderOptions { struct PreloadDocumentFinderOptions {
@ -1873,27 +1876,22 @@ struct PreloadDocumentFinder {
limit: usize, limit: usize,
entry_count: usize, entry_count: usize,
pending_entries: VecDeque<PendingEntry>, pending_entries: VecDeque<PendingEntry>,
disabled_globs: glob::GlobSet, root_dir_entries: Vec<PendingEntry>,
disabled_paths: HashSet<PathBuf>, visited_paths: HashSet<PathBuf>,
} }
impl PreloadDocumentFinder { impl PreloadDocumentFinder {
pub fn new(options: PreloadDocumentFinderOptions) -> Self { pub fn new(options: PreloadDocumentFinderOptions) -> Self {
fn paths_into_globs_and_paths( fn paths_into_globs_and_paths(
input_paths: Vec<PathBuf>, input_paths: Vec<PathBuf>,
) -> (glob::GlobSet, HashSet<PathBuf>) { ) -> glob::PathOrPatternSet {
let mut globs = Vec::with_capacity(input_paths.len()); let mut result = Vec::with_capacity(input_paths.len());
let mut paths = HashSet::with_capacity(input_paths.len());
for path in input_paths { for path in input_paths {
if let Ok(Some(glob)) = if let Ok(path_or_pattern) = glob::PathOrPattern::new(path) {
glob::GlobPattern::new_if_pattern(&path.to_string_lossy()) result.push(path_or_pattern);
{
globs.push(glob);
} else {
paths.insert(path);
} }
} }
(glob::GlobSet::new(globs), paths) glob::PathOrPatternSet::new(result)
} }
fn is_allowed_root_dir(dir_path: &Path) -> bool { fn is_allowed_root_dir(dir_path: &Path) -> bool {
@ -1904,36 +1902,34 @@ impl PreloadDocumentFinder {
true true
} }
let (disabled_globs, disabled_paths) =
paths_into_globs_and_paths(options.disabled_paths);
let mut finder = PreloadDocumentFinder { let mut finder = PreloadDocumentFinder {
limit: options.limit, limit: options.limit,
entry_count: 0, entry_count: 0,
pending_entries: Default::default(), pending_entries: Default::default(),
disabled_globs, root_dir_entries: Default::default(),
disabled_paths, visited_paths: Default::default(),
}; };
let file_patterns = FilePatterns {
include: Some(paths_into_globs_and_paths(options.enabled_paths)),
exclude: paths_into_globs_and_paths(options.disabled_paths),
};
let file_patterns_by_base = file_patterns.split_by_base();
// initialize the finder with the initial paths // initialize the finder with the initial paths
let mut dirs = Vec::with_capacity(options.enabled_paths.len()); for (path, file_patterns) in file_patterns_by_base {
for path in options.enabled_paths { if path.is_dir() {
if !finder.disabled_paths.contains(&path) if is_allowed_root_dir(&path) {
&& !finder.disabled_globs.matches_path(&path)
{
if path.is_dir() {
if is_allowed_root_dir(&path) {
dirs.push(path);
}
} else {
finder finder
.pending_entries .root_dir_entries
.push_back(PendingEntry::SpecifiedRootFile(path)); .push(PendingEntry::Dir(path, Rc::new(file_patterns)));
} }
} else {
finder
.pending_entries
.push_back(PendingEntry::SpecifiedRootFile(path));
} }
} }
for dir in sort_and_remove_non_leaf_dirs(dirs) {
finder.pending_entries.push_back(PendingEntry::Dir(dir));
}
finder finder
} }
@ -2015,48 +2011,60 @@ impl Iterator for PreloadDocumentFinder {
} }
} }
while let Some(entry) = self.pending_entries.pop_front() { // This first drains all the pending entries then adds the root dir entries
match entry { // one at a time to the pending entries before draining them. This is because
PendingEntry::SpecifiedRootFile(file) => { // we're traversing based on directory depth, so we want to search deeper
// since it was a file that was specified as a root url, only // directories first
// verify that it's valid while !self.pending_entries.is_empty() || !self.root_dir_entries.is_empty()
if let Some(specifier) = Self::get_valid_specifier(&file) { {
return Some(specifier); while let Some(entry) = self.pending_entries.pop_front() {
} match entry {
} PendingEntry::SpecifiedRootFile(file) => {
PendingEntry::Dir(dir_path) => { // since it was a file that was specified as a root url, only
if let Ok(read_dir) = fs::read_dir(&dir_path) { // verify that it's valid
self if let Some(specifier) = Self::get_valid_specifier(&file) {
.pending_entries return Some(specifier);
.push_back(PendingEntry::ReadDir(Box::new(read_dir)));
}
}
PendingEntry::ReadDir(mut entries) => {
while let Some(entry) = entries.next() {
self.entry_count += 1;
if self.hit_limit() {
self.pending_entries.clear(); // stop searching
return None;
} }
}
PendingEntry::Dir(dir_path, file_patterns) => {
if self.visited_paths.insert(dir_path.clone()) {
if let Ok(read_dir) = fs::read_dir(&dir_path) {
self.pending_entries.push_back(PendingEntry::ReadDir(
Box::new(read_dir),
file_patterns,
));
}
}
}
PendingEntry::ReadDir(mut entries, file_patterns) => {
while let Some(entry) = entries.next() {
self.entry_count += 1;
if let Ok(entry) = entry { if self.hit_limit() {
let path = entry.path(); self.pending_entries.clear(); // stop searching
if let Ok(file_type) = entry.file_type() { return None;
if !self.disabled_paths.contains(&path) }
&& !self.disabled_globs.matches_path(&path)
{ if let Ok(entry) = entry {
if file_type.is_dir() && is_discoverable_dir(&path) { let path = entry.path();
self if let Ok(file_type) = entry.file_type() {
.pending_entries if file_patterns.matches_path(&path) {
.push_back(PendingEntry::Dir(path.to_path_buf())); if file_type.is_dir() && is_discoverable_dir(&path) {
} else if file_type.is_file() && is_discoverable_file(&path) { self.pending_entries.push_back(PendingEntry::Dir(
if let Some(specifier) = Self::get_valid_specifier(&path) { path.to_path_buf(),
// restore the next entries for next time file_patterns.clone(),
self ));
.pending_entries } else if file_type.is_file() && is_discoverable_file(&path)
.push_front(PendingEntry::ReadDir(entries)); {
return Some(specifier); if let Some(specifier) = Self::get_valid_specifier(&path)
{
// restore the next entries for next time
self.pending_entries.push_front(PendingEntry::ReadDir(
entries,
file_patterns.clone(),
));
return Some(specifier);
}
} }
} }
} }
@ -2065,31 +2073,16 @@ impl Iterator for PreloadDocumentFinder {
} }
} }
} }
if let Some(entry) = self.root_dir_entries.pop() {
self.pending_entries.push_back(entry);
}
} }
None None
} }
} }
/// Removes any directories that are a descendant of another directory in the collection.
fn sort_and_remove_non_leaf_dirs(mut dirs: Vec<PathBuf>) -> Vec<PathBuf> {
if dirs.is_empty() {
return dirs;
}
dirs.sort();
if !dirs.is_empty() {
for i in (0..dirs.len() - 1).rev() {
let prev = &dirs[i + 1];
if prev.starts_with(&dirs[i]) {
dirs.remove(i + 1);
}
}
}
dirs
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::cache::GlobalHttpCache; use crate::cache::GlobalHttpCache;
@ -2435,29 +2428,4 @@ console.log(b, "hello deno");
assert_eq!(paths, vec![]); assert_eq!(paths, vec![]);
} }
} }
#[test]
fn test_sort_and_remove_non_leaf_dirs() {
fn run_test(paths: Vec<&str>, expected_output: Vec<&str>) {
let paths = sort_and_remove_non_leaf_dirs(
paths.into_iter().map(PathBuf::from).collect(),
);
let dirs: Vec<_> =
paths.iter().map(|dir| dir.to_string_lossy()).collect();
assert_eq!(dirs, expected_output);
}
run_test(
vec![
"/test/asdf/test/asdf/",
"/test/asdf/test/asdf/test.ts",
"/test/asdf/",
"/test/asdf/",
"/testing/456/893/",
"/testing/456/893/test/",
],
vec!["/test/asdf/", "/testing/456/893/"],
);
run_test(vec![], vec![]);
}
} }

6
cli/lsp/language_server.rs
View file

@ -1046,10 +1046,12 @@ impl Inner {
self.fmt_options = Default::default(); self.fmt_options = Default::default();
self.lint_options = Default::default(); self.lint_options = Default::default();
if let Some(config_file) = self.get_config_file()? { if let Some(config_file) = self.get_config_file()? {
// this doesn't need to be an actual directory because flags is specified as `None`
let dummy_args_cwd = PathBuf::from("/");
let lint_options = config_file let lint_options = config_file
.to_lint_config() .to_lint_config()
.and_then(|maybe_lint_config| { .and_then(|maybe_lint_config| {
LintOptions::resolve(maybe_lint_config, None) LintOptions::resolve(maybe_lint_config, None, &dummy_args_cwd)
}) })
.map_err(|err| { .map_err(|err| {
anyhow!("Unable to update lint configuration: {:?}", err) anyhow!("Unable to update lint configuration: {:?}", err)
@ -1057,7 +1059,7 @@ impl Inner {
let fmt_options = config_file let fmt_options = config_file
.to_fmt_config() .to_fmt_config()
.and_then(|maybe_fmt_config| { .and_then(|maybe_fmt_config| {
FmtOptions::resolve(maybe_fmt_config, None) FmtOptions::resolve(maybe_fmt_config, None, &dummy_args_cwd)
}) })
.map_err(|err| { .map_err(|err| {
anyhow!("Unable to update formatter configuration: {:?}", err) anyhow!("Unable to update formatter configuration: {:?}", err)

48
cli/module_loader.rs
View file

@ -224,7 +224,7 @@ impl ModuleLoadPreparer {
) -> Result<(), AnyError> { ) -> Result<(), AnyError> {
let lib = self.options.ts_type_lib_window(); let lib = self.options.ts_type_lib_window();
let specifiers = self.collect_specifiers(files); let specifiers = self.collect_specifiers(files)?;
self self
.prepare_module_load( .prepare_module_load(
specifiers, specifiers,
@ -235,28 +235,30 @@ impl ModuleLoadPreparer {
.await .await
} }
fn collect_specifiers(&self, files: &[String]) -> Vec<ModuleSpecifier> { fn collect_specifiers(
let excludes = match self.options.resolve_check_options() { &self,
Ok(o) => o.exclude, files: &[String],
Err(_) => vec![], ) -> Result<Vec<ModuleSpecifier>, AnyError> {
}; let excludes = self.options.resolve_config_excludes()?;
files Ok(
.iter() files
.filter_map(|file| { .iter()
let file_url = .filter_map(|file| {
resolve_url_or_path(file, self.options.initial_cwd()).ok()?; let file_url =
if file_url.scheme() != "file" { resolve_url_or_path(file, self.options.initial_cwd()).ok()?;
return Some(file_url); if file_url.scheme() != "file" {
} return Some(file_url);
// ignore local files that match any of files listed in `exclude` option }
let file_path = file_url.to_file_path().ok()?; // ignore local files that match any of files listed in `exclude` option
if excludes.iter().any(|e| file_path.starts_with(e)) { let file_path = file_url.to_file_path().ok()?;
None if excludes.matches_path(&file_path) {
} else { None
Some(file_url) } else {
} Some(file_url)
}) }
.collect::<Vec<_>>() })
.collect::<Vec<_>>(),
)
} }
} }

2
cli/tests/testdata/doc/invalid_url.out vendored
View file

@ -1,4 +1,4 @@
error: invalid URL: invalid domain character error: Invalid URL 'https://raw.githubusercontent.com%2Fdyedgreen%2Fdeno-sqlite%2Frework_api%2Fmod.ts'
Caused by: Caused by:
invalid domain character invalid domain character

56
cli/tools/bench/mod.rs
View file

@ -15,6 +15,8 @@ use crate::tools::test::format_test_error;
use crate::tools::test::TestFilter; use crate::tools::test::TestFilter;
use crate::util::file_watcher; use crate::util::file_watcher;
use crate::util::fs::collect_specifiers; use crate::util::fs::collect_specifiers;
use crate::util::glob::FilePatterns;
use crate::util::glob::PathOrPattern;
use crate::util::path::is_script_ext; use crate::util::path::is_script_ext;
use crate::version::get_user_agent; use crate::version::get_user_agent;
use crate::worker::CliMainWorkerFactory; use crate::worker::CliMainWorkerFactory;
@ -393,13 +395,33 @@ async fn bench_specifiers(
} }
/// Checks if the path has a basename and extension Deno supports for benches. /// Checks if the path has a basename and extension Deno supports for benches.
fn is_supported_bench_path(path: &Path) -> bool { fn is_supported_bench_path(path: &Path, patterns: &FilePatterns) -> bool {
if !is_script_ext(path) {
false
} else if has_supported_bench_path_name(path) {
true
} else {
// allow someone to explicitly specify a path
let matches_exact_path_or_pattern = patterns
.include
.as_ref()
.map(|p| {
p.inner().iter().any(|p| match p {
PathOrPattern::Path(p) => p == path,
PathOrPattern::Pattern(p) => p.matches_path(path),
})
})
.unwrap_or(false);
matches_exact_path_or_pattern
}
}
fn has_supported_bench_path_name(path: &Path) -> bool {
if let Some(name) = path.file_stem() { if let Some(name) = path.file_stem() {
let basename = name.to_string_lossy(); let basename = name.to_string_lossy();
(basename.ends_with("_bench") basename.ends_with("_bench")
|| basename.ends_with(".bench") || basename.ends_with(".bench")
|| basename == "bench") || basename == "bench"
&& is_script_ext(path)
} else { } else {
false false
} }
@ -420,7 +442,7 @@ pub async fn run_benchmarks(
Permissions::from_options(&cli_options.permissions_options())?; Permissions::from_options(&cli_options.permissions_options())?;
let specifiers = let specifiers =
collect_specifiers(&bench_options.files, is_supported_bench_path)?; collect_specifiers(bench_options.files, is_supported_bench_path)?;
if specifiers.is_empty() { if specifiers.is_empty() {
return Err(generic_error("No bench modules found")); return Err(generic_error("No bench modules found"));
@ -480,16 +502,21 @@ pub async fn run_benchmarks_with_watch(
let bench_options = cli_options.resolve_bench_options(bench_flags)?; let bench_options = cli_options.resolve_bench_options(bench_flags)?;
let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); let _ = watcher_communicator.watch_paths(cli_options.watch_paths());
if let Some(include) = &bench_options.files.include { if let Some(set) = &bench_options.files.include {
let _ = watcher_communicator.watch_paths(include.clone()); let watch_paths = set.base_paths();
if !watch_paths.is_empty() {
let _ = watcher_communicator.watch_paths(watch_paths);
}
} }
let graph_kind = cli_options.type_check_mode().as_graph_kind(); let graph_kind = cli_options.type_check_mode().as_graph_kind();
let module_graph_builder = factory.module_graph_builder().await?; let module_graph_builder = factory.module_graph_builder().await?;
let module_load_preparer = factory.module_load_preparer().await?; let module_load_preparer = factory.module_load_preparer().await?;
let bench_modules = let bench_modules = collect_specifiers(
collect_specifiers(&bench_options.files, is_supported_bench_path)?; bench_options.files.clone(),
is_supported_bench_path,
)?;
// Various bench files should not share the same permissions in terms of // Various bench files should not share the same permissions in terms of
// `PermissionsContainer` - otherwise granting/revoking permissions in one // `PermissionsContainer` - otherwise granting/revoking permissions in one
@ -509,16 +536,13 @@ pub async fn run_benchmarks_with_watch(
let bench_modules_to_reload = if let Some(changed_paths) = changed_paths let bench_modules_to_reload = if let Some(changed_paths) = changed_paths
{ {
let changed_specifiers = changed_paths let changed_paths = changed_paths.into_iter().collect::<HashSet<_>>();
.into_iter()
.filter_map(|p| ModuleSpecifier::from_file_path(p).ok())
.collect::<HashSet<_>>();
let mut result = Vec::new(); let mut result = Vec::new();
for bench_module_specifier in bench_modules { for bench_module_specifier in bench_modules {
if has_graph_root_local_dependent_changed( if has_graph_root_local_dependent_changed(
&graph, &graph,
&bench_module_specifier, &bench_module_specifier,
&changed_specifiers, &changed_paths,
) { ) {
result.push(bench_module_specifier.clone()); result.push(bench_module_specifier.clone());
} }
@ -531,8 +555,10 @@ pub async fn run_benchmarks_with_watch(
let worker_factory = let worker_factory =
Arc::new(factory.create_cli_main_worker_factory().await?); Arc::new(factory.create_cli_main_worker_factory().await?);
// todo(THIS PR): why are we collecting specifiers twice in a row?
// Seems like a perf bug.
let specifiers = let specifiers =
collect_specifiers(&bench_options.files, is_supported_bench_path)? collect_specifiers(bench_options.files, is_supported_bench_path)?
.into_iter() .into_iter()
.filter(|specifier| bench_modules_to_reload.contains(specifier)) .filter(|specifier| bench_modules_to_reload.contains(specifier))
.collect::<Vec<ModuleSpecifier>>(); .collect::<Vec<ModuleSpecifier>>();

34
cli/tools/coverage/mod.rs
View file

@ -9,6 +9,8 @@ use crate::npm::CliNpmResolver;
use crate::tools::fmt::format_json; use crate::tools::fmt::format_json;
use crate::tools::test::is_supported_test_path; use crate::tools::test::is_supported_test_path;
use crate::util::fs::FileCollector; use crate::util::fs::FileCollector;
use crate::util::glob::FilePatterns;
use crate::util::glob::PathOrPatternSet;
use crate::util::text_encoding::source_map_from_code; use crate::util::text_encoding::source_map_from_code;
use deno_ast::MediaType; use deno_ast::MediaType;
@ -371,9 +373,23 @@ fn range_to_src_line_index(
fn collect_coverages( fn collect_coverages(
files: FileFlags, files: FileFlags,
initial_cwd: &Path,
) -> Result<Vec<cdp::ScriptCoverage>, AnyError> { ) -> Result<Vec<cdp::ScriptCoverage>, AnyError> {
let files = files.with_absolute_paths(initial_cwd);
let mut coverages: Vec<cdp::ScriptCoverage> = Vec::new(); let mut coverages: Vec<cdp::ScriptCoverage> = Vec::new();
let file_paths = FileCollector::new(|file_path| { let file_patterns = FilePatterns {
include: Some({
let files = if files.include.is_empty() {
vec![initial_cwd.to_path_buf()]
} else {
files.include
};
PathOrPatternSet::from_absolute_paths(files)?
}),
exclude: PathOrPatternSet::from_absolute_paths(files.ignore)
.context("Invalid ignore pattern.")?,
};
let file_paths = FileCollector::new(|file_path, _| {
file_path file_path
.extension() .extension()
.map(|ext| ext == "json") .map(|ext| ext == "json")
@ -382,16 +398,13 @@ fn collect_coverages(
.ignore_git_folder() .ignore_git_folder()
.ignore_node_modules() .ignore_node_modules()
.ignore_vendor_folder() .ignore_vendor_folder()
.add_ignore_paths(&files.ignore) .collect_file_patterns(file_patterns)?;
.collect_files(if files.include.is_empty() {
None
} else {
Some(&files.include)
})?;
for file_path in file_paths { for file_path in file_paths {
let json = fs::read_to_string(file_path.as_path())?; let new_coverage = fs::read_to_string(file_path.as_path())
let new_coverage: cdp::ScriptCoverage = serde_json::from_str(&json)?; .map_err(AnyError::from)
.and_then(|json| serde_json::from_str(&json).map_err(AnyError::from))
.with_context(|| format!("Failed reading '{}'", file_path.display()))?;
coverages.push(new_coverage); coverages.push(new_coverage);
} }
@ -451,7 +464,8 @@ pub async fn cover_files(
// Use the first include path as the default output path. // Use the first include path as the default output path.
let coverage_root = coverage_flags.files.include[0].clone(); let coverage_root = coverage_flags.files.include[0].clone();
let script_coverages = collect_coverages(coverage_flags.files)?; let script_coverages =
collect_coverages(coverage_flags.files, cli_options.initial_cwd())?;
if script_coverages.is_empty() { if script_coverages.is_empty() {
return Err(generic_error("No coverage files found")); return Err(generic_error("No coverage files found"));
} }

40
cli/tools/doc.rs
View file

@ -12,12 +12,13 @@ use crate::factory::CliFactory;
use crate::graph_util::graph_lock_or_exit; use crate::graph_util::graph_lock_or_exit;
use crate::graph_util::CreateGraphOptions; use crate::graph_util::CreateGraphOptions;
use crate::tsc::get_types_declaration_file_text; use crate::tsc::get_types_declaration_file_text;
use crate::util::glob::expand_globs; use crate::util::fs::collect_specifiers;
use crate::util::glob::FilePatterns;
use crate::util::glob::PathOrPatternSet;
use deno_core::anyhow::bail; use deno_core::anyhow::bail;
use deno_core::anyhow::Context; use deno_core::anyhow::Context;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_core::futures::FutureExt; use deno_core::futures::FutureExt;
use deno_core::resolve_url_or_path;
use deno_doc as doc; use deno_doc as doc;
use deno_graph::CapturingModuleParser; use deno_graph::CapturingModuleParser;
use deno_graph::DefaultParsedSourceStore; use deno_graph::DefaultParsedSourceStore;
@ -100,19 +101,28 @@ pub async fn doc(flags: Flags, doc_flags: DocFlags) -> Result<(), AnyError> {
let module_graph_builder = factory.module_graph_builder().await?; let module_graph_builder = factory.module_graph_builder().await?;
let maybe_lockfile = factory.maybe_lockfile(); let maybe_lockfile = factory.maybe_lockfile();
let expanded_globs = let module_specifiers = collect_specifiers(
expand_globs(source_files.iter().map(PathBuf::from).collect())?; FilePatterns {
let module_specifiers: Result<Vec<ModuleSpecifier>, AnyError> = include: Some(PathOrPatternSet::from_absolute_paths(
expanded_globs source_files
.iter() .iter()
.map(|source_file| { .map(|p| {
Ok(resolve_url_or_path( if p.starts_with("https:")
&source_file.to_string_lossy(), || p.starts_with("http:")
cli_options.initial_cwd(), || p.starts_with("file:")
)?) {
}) // todo(dsherret): don't store URLs in PathBufs
.collect(); PathBuf::from(p)
let module_specifiers = module_specifiers?; } else {
cli_options.initial_cwd().join(p)
}
})
.collect(),
)?),
exclude: Default::default(),
},
|_, _| true,
)?;
let mut loader = module_graph_builder.create_graph_loader(); let mut loader = module_graph_builder.create_graph_loader();
let graph = module_graph_builder let graph = module_graph_builder
.create_graph_with_options(CreateGraphOptions { .create_graph_with_options(CreateGraphOptions {

39
cli/tools/fmt.rs
View file

@ -8,7 +8,6 @@
//! the same functions as ops available in JS runtime. //! the same functions as ops available in JS runtime.
use crate::args::CliOptions; use crate::args::CliOptions;
use crate::args::FilesConfig;
use crate::args::Flags; use crate::args::Flags;
use crate::args::FmtFlags; use crate::args::FmtFlags;
use crate::args::FmtOptions; use crate::args::FmtOptions;
@ -18,7 +17,9 @@ use crate::colors;
use crate::factory::CliFactory; use crate::factory::CliFactory;
use crate::util::diff::diff; use crate::util::diff::diff;
use crate::util::file_watcher; use crate::util::file_watcher;
use crate::util::fs::canonicalize_path;
use crate::util::fs::FileCollector; use crate::util::fs::FileCollector;
use crate::util::glob::FilePatterns;
use crate::util::path::get_extension; use crate::util::path::get_extension;
use crate::util::text_encoding; use crate::util::text_encoding;
use deno_ast::ParsedSource; use deno_ast::ParsedSource;
@ -72,7 +73,7 @@ pub async fn format(flags: Flags, fmt_flags: FmtFlags) -> Result<(), AnyError> {
let cli_options = factory.cli_options(); let cli_options = factory.cli_options();
let fmt_options = cli_options.resolve_fmt_options(fmt_flags)?; let fmt_options = cli_options.resolve_fmt_options(fmt_flags)?;
let files = let files =
collect_fmt_files(&fmt_options.files).and_then(|files| { collect_fmt_files(fmt_options.files.clone()).and_then(|files| {
if files.is_empty() { if files.is_empty() {
Err(generic_error("No target files found.")) Err(generic_error("No target files found."))
} else { } else {
@ -85,13 +86,21 @@ pub async fn format(flags: Flags, fmt_flags: FmtFlags) -> Result<(), AnyError> {
// check all files on any changed (https://github.com/denoland/deno/issues/12446) // check all files on any changed (https://github.com/denoland/deno/issues/12446)
files files
.iter() .iter()
.any(|path| paths.contains(path)) .any(|path| {
canonicalize_path(path)
.map(|path| paths.contains(&path))
.unwrap_or(false)
})
.then_some(files) .then_some(files)
.unwrap_or_else(|| [].to_vec()) .unwrap_or_else(|| [].to_vec())
} else { } else {
files files
.into_iter() .into_iter()
.filter(|path| paths.contains(path)) .filter(|path| {
canonicalize_path(path)
.map(|path| paths.contains(&path))
.unwrap_or(false)
})
.collect::<Vec<_>>() .collect::<Vec<_>>()
} }
} else { } else {
@ -108,13 +117,14 @@ pub async fn format(flags: Flags, fmt_flags: FmtFlags) -> Result<(), AnyError> {
let factory = CliFactory::from_flags(flags).await?; let factory = CliFactory::from_flags(flags).await?;
let cli_options = factory.cli_options(); let cli_options = factory.cli_options();
let fmt_options = cli_options.resolve_fmt_options(fmt_flags)?; let fmt_options = cli_options.resolve_fmt_options(fmt_flags)?;
let files = collect_fmt_files(&fmt_options.files).and_then(|files| { let files =
if files.is_empty() { collect_fmt_files(fmt_options.files.clone()).and_then(|files| {
Err(generic_error("No target files found.")) if files.is_empty() {
} else { Err(generic_error("No target files found."))
Ok(files) } else {
} Ok(files)
})?; }
})?;
format_files(factory, fmt_options, files).await?; format_files(factory, fmt_options, files).await?;
} }
@ -144,13 +154,12 @@ async fn format_files(
Ok(()) Ok(())
} }
fn collect_fmt_files(files: &FilesConfig) -> Result<Vec<PathBuf>, AnyError> { fn collect_fmt_files(files: FilePatterns) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(is_supported_ext_fmt) FileCollector::new(|path, _| is_supported_ext_fmt(path))
.ignore_git_folder() .ignore_git_folder()
.ignore_node_modules() .ignore_node_modules()
.ignore_vendor_folder() .ignore_vendor_folder()
.add_ignore_paths(&files.exclude) .collect_file_patterns(files)
.collect_files(files.include.as_deref())
} }
/// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks /// Formats markdown (using <https://github.com/dprint/dprint-plugin-markdown>) and its code blocks

38
cli/tools/lint.rs
View file

@ -2,7 +2,6 @@
//! This module provides file linting utilities using //! This module provides file linting utilities using
//! [`deno_lint`](https://github.com/denoland/deno_lint). //! [`deno_lint`](https://github.com/denoland/deno_lint).
use crate::args::FilesConfig;
use crate::args::Flags; use crate::args::Flags;
use crate::args::LintFlags; use crate::args::LintFlags;
use crate::args::LintOptions; use crate::args::LintOptions;
@ -12,7 +11,9 @@ use crate::colors;
use crate::factory::CliFactory; use crate::factory::CliFactory;
use crate::tools::fmt::run_parallelized; use crate::tools::fmt::run_parallelized;
use crate::util::file_watcher; use crate::util::file_watcher;
use crate::util::fs::canonicalize_path;
use crate::util::fs::FileCollector; use crate::util::fs::FileCollector;
use crate::util::glob::FilePatterns;
use crate::util::path::is_script_ext; use crate::util::path::is_script_ext;
use crate::util::sync::AtomicFlag; use crate::util::sync::AtomicFlag;
use deno_ast::MediaType; use deno_ast::MediaType;
@ -66,21 +67,26 @@ pub async fn lint(flags: Flags, lint_flags: LintFlags) -> Result<(), AnyError> {
let factory = CliFactory::from_flags(flags).await?; let factory = CliFactory::from_flags(flags).await?;
let cli_options = factory.cli_options(); let cli_options = factory.cli_options();
let lint_options = cli_options.resolve_lint_options(lint_flags)?; let lint_options = cli_options.resolve_lint_options(lint_flags)?;
let files = let files = collect_lint_files(lint_options.files.clone()).and_then(
collect_lint_files(&lint_options.files).and_then(|files| { |files| {
if files.is_empty() { if files.is_empty() {
Err(generic_error("No target files found.")) Err(generic_error("No target files found."))
} else { } else {
Ok(files) Ok(files)
} }
})?; },
)?;
_ = watcher_communicator.watch_paths(files.clone()); _ = watcher_communicator.watch_paths(files.clone());
let lint_paths = if let Some(paths) = changed_paths { let lint_paths = if let Some(paths) = changed_paths {
// lint all files on any changed (https://github.com/denoland/deno/issues/12446) // lint all files on any changed (https://github.com/denoland/deno/issues/12446)
files files
.iter() .iter()
.any(|path| paths.contains(path)) .any(|path| {
canonicalize_path(path)
.map(|p| paths.contains(&p))
.unwrap_or(false)
})
.then_some(files) .then_some(files)
.unwrap_or_else(|| [].to_vec()) .unwrap_or_else(|| [].to_vec())
} else { } else {
@ -109,13 +115,14 @@ pub async fn lint(flags: Flags, lint_flags: LintFlags) -> Result<(), AnyError> {
reporter_lock.lock().unwrap().close(1); reporter_lock.lock().unwrap().close(1);
success success
} else { } else {
let target_files = collect_lint_files(files).and_then(|files| { let target_files =
if files.is_empty() { collect_lint_files(files.clone()).and_then(|files| {
Err(generic_error("No target files found.")) if files.is_empty() {
} else { Err(generic_error("No target files found."))
Ok(files) } else {
} Ok(files)
})?; }
})?;
debug!("Found {} files", target_files.len()); debug!("Found {} files", target_files.len());
lint_files(factory, lint_options, target_files).await? lint_files(factory, lint_options, target_files).await?
}; };
@ -191,13 +198,12 @@ async fn lint_files(
Ok(!has_error.is_raised()) Ok(!has_error.is_raised())
} }
fn collect_lint_files(files: &FilesConfig) -> Result<Vec<PathBuf>, AnyError> { fn collect_lint_files(files: FilePatterns) -> Result<Vec<PathBuf>, AnyError> {
FileCollector::new(is_script_ext) FileCollector::new(|path, _| is_script_ext(path))
.ignore_git_folder() .ignore_git_folder()
.ignore_node_modules() .ignore_node_modules()
.ignore_vendor_folder() .ignore_vendor_folder()
.add_ignore_paths(&files.exclude) .collect_file_patterns(files)
.collect_files(files.include.as_deref())
} }
pub fn print_rules_list(json: bool, maybe_rules_tags: Option<Vec<String>>) { pub fn print_rules_list(json: bool, maybe_rules_tags: Option<Vec<String>>) {

75
cli/tools/test/mod.rs
View file

@ -1,7 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use crate::args::CliOptions; use crate::args::CliOptions;
use crate::args::FilesConfig;
use crate::args::Flags; use crate::args::Flags;
use crate::args::TestFlags; use crate::args::TestFlags;
use crate::args::TestReporterConfig; use crate::args::TestReporterConfig;
@ -17,6 +16,8 @@ use crate::module_loader::ModuleLoadPreparer;
use crate::ops; use crate::ops;
use crate::util::file_watcher; use crate::util::file_watcher;
use crate::util::fs::collect_specifiers; use crate::util::fs::collect_specifiers;
use crate::util::glob::FilePatterns;
use crate::util::glob::PathOrPattern;
use crate::util::path::get_extension; use crate::util::path::get_extension;
use crate::util::path::is_script_ext; use crate::util::path::is_script_ext;
use crate::util::path::mapped_specifier_for_tsc; use crate::util::path::mapped_specifier_for_tsc;
@ -1048,14 +1049,41 @@ pub async fn report_tests(
(Ok(()), receiver) (Ok(()), receiver)
} }
fn is_supported_test_path_predicate(
path: &Path,
patterns: &FilePatterns,
) -> bool {
if !is_script_ext(path) {
false
} else if has_supported_test_path_name(path) {
true
} else {
// allow someone to explicitly specify a path
let matches_exact_path_or_pattern = patterns
.include
.as_ref()
.map(|p| {
p.inner().iter().any(|p| match p {
PathOrPattern::Path(p) => p == path,
PathOrPattern::Pattern(p) => p.matches_path(path),
})
})
.unwrap_or(false);
matches_exact_path_or_pattern
}
}
/// Checks if the path has a basename and extension Deno supports for tests. /// Checks if the path has a basename and extension Deno supports for tests.
pub(crate) fn is_supported_test_path(path: &Path) -> bool { pub(crate) fn is_supported_test_path(path: &Path) -> bool {
has_supported_test_path_name(path) && is_script_ext(path)
}
fn has_supported_test_path_name(path: &Path) -> bool {
if let Some(name) = path.file_stem() { if let Some(name) = path.file_stem() {
let basename = name.to_string_lossy(); let basename = name.to_string_lossy();
(basename.ends_with("_test") basename.ends_with("_test")
|| basename.ends_with(".test") || basename.ends_with(".test")
|| basename == "test") || basename == "test"
&& is_script_ext(path)
} else { } else {
false false
} }
@ -1094,13 +1122,15 @@ fn is_supported_test_ext(path: &Path) -> bool {
/// - Specifiers matching the `is_supported_test_path` are marked as `TestMode::Executable`. /// - Specifiers matching the `is_supported_test_path` are marked as `TestMode::Executable`.
/// - Specifiers matching both predicates are marked as `TestMode::Both` /// - Specifiers matching both predicates are marked as `TestMode::Both`
fn collect_specifiers_with_test_mode( fn collect_specifiers_with_test_mode(
files: &FilesConfig, files: FilePatterns,
include_inline: &bool, include_inline: &bool,
) -> Result<Vec<(ModuleSpecifier, TestMode)>, AnyError> { ) -> Result<Vec<(ModuleSpecifier, TestMode)>, AnyError> {
let module_specifiers = collect_specifiers(files, is_supported_test_path)?; // todo(dsherret): there's no need to collect twice as it's slow
let module_specifiers =
collect_specifiers(files.clone(), is_supported_test_path_predicate)?;
if *include_inline { if *include_inline {
return collect_specifiers(files, is_supported_test_ext).map( return collect_specifiers(files, |p, _| is_supported_test_ext(p)).map(
|specifiers| { |specifiers| {
specifiers specifiers
.into_iter() .into_iter()
@ -1136,7 +1166,7 @@ fn collect_specifiers_with_test_mode(
/// as well. /// as well.
async fn fetch_specifiers_with_test_mode( async fn fetch_specifiers_with_test_mode(
file_fetcher: &FileFetcher, file_fetcher: &FileFetcher,
files: &FilesConfig, files: FilePatterns,
doc: &bool, doc: &bool,
) -> Result<Vec<(ModuleSpecifier, TestMode)>, AnyError> { ) -> Result<Vec<(ModuleSpecifier, TestMode)>, AnyError> {
let mut specifiers_with_mode = collect_specifiers_with_test_mode(files, doc)?; let mut specifiers_with_mode = collect_specifiers_with_test_mode(files, doc)?;
@ -1174,7 +1204,7 @@ pub async fn run_tests(
let specifiers_with_mode = fetch_specifiers_with_test_mode( let specifiers_with_mode = fetch_specifiers_with_test_mode(
file_fetcher, file_fetcher,
&test_options.files, test_options.files.clone(),
&test_options.doc, &test_options.doc,
) )
.await?; .await?;
@ -1264,8 +1294,11 @@ pub async fn run_tests_with_watch(
let test_options = cli_options.resolve_test_options(test_flags)?; let test_options = cli_options.resolve_test_options(test_flags)?;
let _ = watcher_communicator.watch_paths(cli_options.watch_paths()); let _ = watcher_communicator.watch_paths(cli_options.watch_paths());
if let Some(include) = &test_options.files.include { if let Some(set) = &test_options.files.include {
let _ = watcher_communicator.watch_paths(include.clone()); let watch_paths = set.base_paths();
if !watch_paths.is_empty() {
let _ = watcher_communicator.watch_paths(watch_paths);
}
} }
let graph_kind = cli_options.type_check_mode().as_graph_kind(); let graph_kind = cli_options.type_check_mode().as_graph_kind();
@ -1274,13 +1307,18 @@ pub async fn run_tests_with_watch(
let module_graph_builder = factory.module_graph_builder().await?; let module_graph_builder = factory.module_graph_builder().await?;
let file_fetcher = factory.file_fetcher()?; let file_fetcher = factory.file_fetcher()?;
let test_modules = if test_options.doc { let test_modules = if test_options.doc {
collect_specifiers(&test_options.files, is_supported_test_ext) collect_specifiers(test_options.files.clone(), |p, _| {
is_supported_test_ext(p)
})
} else { } else {
collect_specifiers(&test_options.files, is_supported_test_path) collect_specifiers(
test_options.files.clone(),
is_supported_test_path_predicate,
)
}?; }?;
let permissions = let permissions =
Permissions::from_options(&cli_options.permissions_options())?; Permissions::from_options(&cli_options.permissions_options())?;
let graph = module_graph_builder let graph = module_graph_builder
.create_graph(graph_kind, test_modules.clone()) .create_graph(graph_kind, test_modules.clone())
.await?; .await?;
@ -1293,16 +1331,13 @@ pub async fn run_tests_with_watch(
let test_modules_to_reload = if let Some(changed_paths) = changed_paths let test_modules_to_reload = if let Some(changed_paths) = changed_paths
{ {
let changed_specifiers = changed_paths
.into_iter()
.filter_map(|p| ModuleSpecifier::from_file_path(p).ok())
.collect::<HashSet<_>>();
let mut result = Vec::new(); let mut result = Vec::new();
let changed_paths = changed_paths.into_iter().collect::<HashSet<_>>();
for test_module_specifier in test_modules { for test_module_specifier in test_modules {
if has_graph_root_local_dependent_changed( if has_graph_root_local_dependent_changed(
&graph, &graph,
&test_module_specifier, &test_module_specifier,
&changed_specifiers, &changed_paths,
) { ) {
result.push(test_module_specifier.clone()); result.push(test_module_specifier.clone());
} }
@ -1317,7 +1352,7 @@ pub async fn run_tests_with_watch(
let module_load_preparer = factory.module_load_preparer().await?; let module_load_preparer = factory.module_load_preparer().await?;
let specifiers_with_mode = fetch_specifiers_with_test_mode( let specifiers_with_mode = fetch_specifiers_with_test_mode(
file_fetcher, file_fetcher,
&test_options.files, test_options.files.clone(),
&test_options.doc, &test_options.doc,
) )
.await? .await?

358
cli/util/fs.rs
View file

@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use deno_core::anyhow::anyhow;
use deno_core::anyhow::Context; use deno_core::anyhow::Context;
use deno_core::error::AnyError; use deno_core::error::AnyError;
pub use deno_core::normalize_path; pub use deno_core::normalize_path;
@ -8,7 +9,7 @@ use deno_core::ModuleSpecifier;
use deno_runtime::deno_crypto::rand; use deno_runtime::deno_crypto::rand;
use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::FileSystem;
use deno_runtime::deno_node::PathClean; use deno_runtime::deno_node::PathClean;
use std::borrow::Cow; use std::collections::HashSet;
use std::env::current_dir; use std::env::current_dir;
use std::fmt::Write as FmtWrite; use std::fmt::Write as FmtWrite;
use std::fs::OpenOptions; use std::fs::OpenOptions;
@ -21,11 +22,13 @@ use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
use walkdir::WalkDir; use walkdir::WalkDir;
use crate::args::FilesConfig;
use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBar;
use crate::util::progress_bar::ProgressBarStyle; use crate::util::progress_bar::ProgressBarStyle;
use crate::util::progress_bar::ProgressMessagePrompt; use crate::util::progress_bar::ProgressMessagePrompt;
use super::glob::FilePatterns;
use super::glob::PathOrPattern;
use super::glob::PathOrPatternSet;
use super::path::specifier_to_file_path; use super::path::specifier_to_file_path;
/// Writes the file to the file system at a temporary path, then /// Writes the file to the file system at a temporary path, then
@ -244,18 +247,16 @@ pub fn resolve_from_cwd(path: &Path) -> Result<PathBuf, AnyError> {
/// Collects file paths that satisfy the given predicate, by recursively walking `files`. /// Collects file paths that satisfy the given predicate, by recursively walking `files`.
/// If the walker visits a path that is listed in `ignore`, it skips descending into the directory. /// If the walker visits a path that is listed in `ignore`, it skips descending into the directory.
pub struct FileCollector<TFilter: Fn(&Path) -> bool> { pub struct FileCollector<TFilter: Fn(&Path, &FilePatterns) -> bool> {
canonicalized_ignore: Vec<PathBuf>,
file_filter: TFilter, file_filter: TFilter,
ignore_git_folder: bool, ignore_git_folder: bool,
ignore_node_modules: bool, ignore_node_modules: bool,
ignore_vendor_folder: bool, ignore_vendor_folder: bool,
} }
impl<TFilter: Fn(&Path) -> bool> FileCollector<TFilter> { impl<TFilter: Fn(&Path, &FilePatterns) -> bool> FileCollector<TFilter> {
pub fn new(file_filter: TFilter) -> Self { pub fn new(file_filter: TFilter) -> Self {
Self { Self {
canonicalized_ignore: Default::default(),
file_filter, file_filter,
ignore_git_folder: false, ignore_git_folder: false,
ignore_node_modules: false, ignore_node_modules: false,
@ -263,14 +264,6 @@ impl<TFilter: Fn(&Path) -> bool> FileCollector<TFilter> {
} }
} }
pub fn add_ignore_paths(mut self, paths: &[PathBuf]) -> Self {
// retain only the paths which exist and ignore the rest
self
.canonicalized_ignore
.extend(paths.iter().filter_map(|i| canonicalize_path(i).ok()));
self
}
pub fn ignore_node_modules(mut self) -> Self { pub fn ignore_node_modules(mut self) -> Self {
self.ignore_node_modules = true; self.ignore_node_modules = true;
self self
@ -286,58 +279,62 @@ impl<TFilter: Fn(&Path) -> bool> FileCollector<TFilter> {
self self
} }
pub fn collect_files( pub fn collect_file_patterns(
&self, &self,
files: Option<&[PathBuf]>, file_patterns: FilePatterns,
) -> Result<Vec<PathBuf>, AnyError> { ) -> Result<Vec<PathBuf>, AnyError> {
let mut target_files = Vec::new(); let mut target_files = Vec::new();
let files = if let Some(files) = files { let mut visited_paths = HashSet::new();
Cow::Borrowed(files) let file_patterns_by_base = file_patterns.split_by_base();
} else { for (base, file_patterns) in file_patterns_by_base {
Cow::Owned(vec![PathBuf::from(".")]) let file = normalize_path(base);
}; // use an iterator in order to minimize the number of file system operations
for file in files.iter() { let mut iterator = WalkDir::new(&file)
if let Ok(file) = canonicalize_path(file) { .follow_links(false) // the default, but be explicit
// use an iterator like this in order to minimize the number of file system operations .into_iter();
let mut iterator = WalkDir::new(&file).into_iter(); loop {
loop { let e = match iterator.next() {
let e = match iterator.next() { None => break,
None => break, Some(Err(_)) => continue,
Some(Err(_)) => continue, Some(Ok(entry)) => entry,
Some(Ok(entry)) => entry, };
}; let file_type = e.file_type();
let file_type = e.file_type(); let is_dir = file_type.is_dir();
let is_dir = file_type.is_dir(); let c = e.path().to_path_buf();
if let Ok(c) = canonicalize_path(e.path()) { if file_patterns.exclude.matches_path(&c)
if self.canonicalized_ignore.iter().any(|i| c.starts_with(i)) { || !is_dir
if is_dir { && !file_patterns
iterator.skip_current_dir(); .include
} .as_ref()
} else if is_dir { .map(|i| i.matches_path(&c))
let should_ignore_dir = c .unwrap_or(true)
.file_name() {
.map(|dir_name| { if is_dir {
let dir_name = dir_name.to_string_lossy().to_lowercase();
let is_ignored_file = match dir_name.as_str() {
"node_modules" => self.ignore_node_modules,
"vendor" => self.ignore_vendor_folder,
".git" => self.ignore_git_folder,
_ => false,
};
// allow the user to opt out of ignoring by explicitly specifying the dir
file != c && is_ignored_file
})
.unwrap_or(false);
if should_ignore_dir {
iterator.skip_current_dir();
}
} else if (self.file_filter)(e.path()) {
target_files.push(c);
}
} else if is_dir {
// failed canonicalizing, so skip it
iterator.skip_current_dir(); iterator.skip_current_dir();
} }
} else if is_dir {
let should_ignore_dir = c
.file_name()
.map(|dir_name| {
let dir_name = dir_name.to_string_lossy().to_lowercase();
let is_ignored_file = match dir_name.as_str() {
"node_modules" => self.ignore_node_modules,
"vendor" => self.ignore_vendor_folder,
".git" => self.ignore_git_folder,
_ => false,
};
// allow the user to opt out of ignoring by explicitly specifying the dir
file != c && is_ignored_file
})
.unwrap_or(false)
|| !visited_paths.insert(c.clone());
if should_ignore_dir {
iterator.skip_current_dir();
}
} else if (self.file_filter)(&c, &file_patterns)
&& visited_paths.insert(c.clone())
{
target_files.push(c);
} }
} }
} }
@ -349,53 +346,67 @@ impl<TFilter: Fn(&Path) -> bool> FileCollector<TFilter> {
/// Specifiers that start with http and https are left intact. /// Specifiers that start with http and https are left intact.
/// Note: This ignores all .git and node_modules folders. /// Note: This ignores all .git and node_modules folders.
pub fn collect_specifiers( pub fn collect_specifiers(
files: &FilesConfig, mut files: FilePatterns,
predicate: impl Fn(&Path) -> bool, predicate: impl Fn(&Path, &FilePatterns) -> bool,
) -> Result<Vec<ModuleSpecifier>, AnyError> { ) -> Result<Vec<ModuleSpecifier>, AnyError> {
let mut prepared = vec![]; let mut prepared = vec![];
let file_collector = FileCollector::new(predicate)
.add_ignore_paths(&files.exclude) // break out the remote specifiers
if let Some(include_mut) = &mut files.include {
let includes = std::mem::take(include_mut);
let path_or_patterns = includes.into_path_or_patterns();
let mut result = Vec::with_capacity(path_or_patterns.len());
for path_or_pattern in path_or_patterns {
match path_or_pattern {
PathOrPattern::Path(path) => {
// todo(dsherret): we should improve this to not store URLs in a PathBuf
let path_str = path.to_string_lossy();
let lowercase_path = path_str.to_lowercase();
if lowercase_path.starts_with("http://")
|| lowercase_path.starts_with("https://")
{
// take out the url
let url = ModuleSpecifier::parse(&path_str)
.with_context(|| format!("Invalid URL '{}'", path_str))?;
prepared.push(url);
} else if lowercase_path.starts_with("file://") {
let url = ModuleSpecifier::parse(&path_str)
.with_context(|| format!("Invalid URL '{}'", path_str))?;
let p = specifier_to_file_path(&url)?;
if p.is_dir() {
result.push(PathOrPattern::Path(p));
} else {
prepared.push(url)
}
} else if path.is_dir() {
result.push(PathOrPattern::Path(path));
} else if !files.exclude.matches_path(&path) {
let url = ModuleSpecifier::from_file_path(&path)
.map_err(|_| anyhow!("Invalid file path '{}'", path.display()))?;
prepared.push(url);
}
}
PathOrPattern::Pattern(pattern) => {
// add it back
result.push(PathOrPattern::Pattern(pattern));
}
}
}
*include_mut = PathOrPatternSet::new(result);
}
let collected_files = FileCollector::new(predicate)
.ignore_git_folder() .ignore_git_folder()
.ignore_node_modules() .ignore_node_modules()
.ignore_vendor_folder(); .ignore_vendor_folder()
.collect_file_patterns(files)?;
let mut collected_files_as_urls = collected_files
.iter()
.map(|f| ModuleSpecifier::from_file_path(f).unwrap())
.collect::<Vec<ModuleSpecifier>>();
let root_path = current_dir()?; collected_files_as_urls.sort();
let include_files = if let Some(include) = &files.include { prepared.extend(collected_files_as_urls);
Cow::Borrowed(include)
} else {
Cow::Owned(vec![root_path.clone()])
};
for path in include_files.iter() {
let path = path.to_string_lossy();
let lowercase_path = path.to_lowercase();
if lowercase_path.starts_with("http://")
|| lowercase_path.starts_with("https://")
{
let url = ModuleSpecifier::parse(&path)?;
prepared.push(url);
continue;
}
let p = if lowercase_path.starts_with("file://") {
specifier_to_file_path(&ModuleSpecifier::parse(&path)?)?
} else {
root_path.join(path.as_ref())
};
let p = normalize_path(p);
if p.is_dir() {
let test_files = file_collector.collect_files(Some(&[p]))?;
let mut test_files_as_urls = test_files
.iter()
.map(|f| ModuleSpecifier::from_file_path(f).unwrap())
.collect::<Vec<ModuleSpecifier>>();
test_files_as_urls.sort();
prepared.extend(test_files_as_urls);
} else {
let url = ModuleSpecifier::from_file_path(p).unwrap();
prepared.push(url);
}
}
Ok(prepared) Ok(prepared)
} }
@ -812,18 +823,29 @@ mod tests {
let ignore_dir_files = ["g.d.ts", ".gitignore"]; let ignore_dir_files = ["g.d.ts", ".gitignore"];
create_files(&ignore_dir_path, &ignore_dir_files); create_files(&ignore_dir_path, &ignore_dir_files);
let file_collector = FileCollector::new(|path| { let file_patterns = FilePatterns {
include: Some(
PathOrPatternSet::from_absolute_paths(
vec![root_dir_path.to_path_buf()],
)
.unwrap(),
),
exclude: PathOrPatternSet::from_absolute_paths(vec![
ignore_dir_path.to_path_buf()
])
.unwrap(),
};
let file_collector = FileCollector::new(|path, _| {
// exclude dotfiles // exclude dotfiles
path path
.file_name() .file_name()
.and_then(|f| f.to_str()) .and_then(|f| f.to_str())
.map(|f| !f.starts_with('.')) .map(|f| !f.starts_with('.'))
.unwrap_or(false) .unwrap_or(false)
}) });
.add_ignore_paths(&[ignore_dir_path.to_path_buf()]);
let result = file_collector let result = file_collector
.collect_files(Some(&[root_dir_path.to_path_buf()])) .collect_file_patterns(file_patterns.clone())
.unwrap(); .unwrap();
let expected = [ let expected = [
"README.md", "README.md",
@ -850,7 +872,7 @@ mod tests {
.ignore_node_modules() .ignore_node_modules()
.ignore_vendor_folder(); .ignore_vendor_folder();
let result = file_collector let result = file_collector
.collect_files(Some(&[root_dir_path.to_path_buf()])) .collect_file_patterns(file_patterns.clone())
.unwrap(); .unwrap();
let expected = [ let expected = [
"README.md", "README.md",
@ -869,12 +891,20 @@ mod tests {
assert_eq!(file_names, expected); assert_eq!(file_names, expected);
// test opting out of ignoring by specifying the dir // test opting out of ignoring by specifying the dir
let result = file_collector let file_patterns = FilePatterns {
.collect_files(Some(&[ include: Some(
root_dir_path.to_path_buf(), PathOrPatternSet::from_absolute_paths(vec![
root_dir_path.to_path_buf().join("child/node_modules/"), root_dir_path.to_path_buf(),
])) root_dir_path.to_path_buf().join("child/node_modules/"),
.unwrap(); ])
.unwrap(),
),
exclude: PathOrPatternSet::from_absolute_paths(vec![
ignore_dir_path.to_path_buf()
])
.unwrap(),
};
let result = file_collector.collect_file_patterns(file_patterns).unwrap();
let expected = [ let expected = [
"README.md", "README.md",
"a.ts", "a.ts",
@ -930,7 +960,7 @@ mod tests {
let ignore_dir_files = ["g.d.ts", ".gitignore"]; let ignore_dir_files = ["g.d.ts", ".gitignore"];
create_files(&ignore_dir_path, &ignore_dir_files); create_files(&ignore_dir_path, &ignore_dir_files);
let predicate = |path: &Path| { let predicate = |path: &Path, _: &FilePatterns| {
// exclude dotfiles // exclude dotfiles
path path
.file_name() .file_name()
@ -940,38 +970,46 @@ mod tests {
}; };
let result = collect_specifiers( let result = collect_specifiers(
&FilesConfig { FilePatterns {
include: Some(vec![ include: Some(
PathBuf::from("http://localhost:8080"), PathOrPatternSet::from_absolute_paths(vec![
root_dir_path.to_path_buf(), PathBuf::from("http://localhost:8080"),
PathBuf::from("https://localhost:8080".to_string()), root_dir_path.to_path_buf(),
]), PathBuf::from("https://localhost:8080".to_string()),
exclude: vec![ignore_dir_path.to_path_buf()], ])
.unwrap(),
),
exclude: PathOrPatternSet::from_absolute_paths(vec![
ignore_dir_path.to_path_buf()
])
.unwrap(),
}, },
predicate, predicate,
) )
.unwrap(); .unwrap();
let root_dir_url = let root_dir_url = ModuleSpecifier::from_file_path(&root_dir_path)
ModuleSpecifier::from_file_path(root_dir_path.canonicalize()) .unwrap()
.unwrap() .to_string();
.to_string(); let expected = vec![
let expected: Vec<ModuleSpecifier> = [ "http://localhost:8080/".to_string(),
"http://localhost:8080", "https://localhost:8080/".to_string(),
&format!("{root_dir_url}/a.ts"), format!("{root_dir_url}/a.ts"),
&format!("{root_dir_url}/b.js"), format!("{root_dir_url}/b.js"),
&format!("{root_dir_url}/c.tsx"), format!("{root_dir_url}/c.tsx"),
&format!("{root_dir_url}/child/README.md"), format!("{root_dir_url}/child/README.md"),
&format!("{root_dir_url}/child/e.mjs"), format!("{root_dir_url}/child/e.mjs"),
&format!("{root_dir_url}/child/f.mjsx"), format!("{root_dir_url}/child/f.mjsx"),
&format!("{root_dir_url}/d.jsx"), format!("{root_dir_url}/d.jsx"),
"https://localhost:8080", ];
]
.iter()
.map(|f| ModuleSpecifier::parse(f).unwrap())
.collect::<Vec<_>>();
assert_eq!(result, expected); assert_eq!(
result
.into_iter()
.map(|s| s.to_string())
.collect::<Vec<_>>(),
expected
);
let scheme = if cfg!(target_os = "windows") { let scheme = if cfg!(target_os = "windows") {
"file:///" "file:///"
@ -979,28 +1017,34 @@ mod tests {
"file://" "file://"
}; };
let result = collect_specifiers( let result = collect_specifiers(
&FilesConfig { FilePatterns {
include: Some(vec![PathBuf::from(format!( include: Some(
"{}{}", PathOrPatternSet::from_absolute_paths(vec![PathBuf::from(format!(
scheme, "{}{}",
root_dir_path.join("child").to_string().replace('\\', "/") scheme,
))]), root_dir_path.join("child").to_string().replace('\\', "/")
exclude: vec![], ))])
.unwrap(),
),
exclude: Default::default(),
}, },
predicate, predicate,
) )
.unwrap(); .unwrap();
let expected: Vec<ModuleSpecifier> = [ let expected = vec![
&format!("{root_dir_url}/child/README.md"), format!("{root_dir_url}/child/README.md"),
&format!("{root_dir_url}/child/e.mjs"), format!("{root_dir_url}/child/e.mjs"),
&format!("{root_dir_url}/child/f.mjsx"), format!("{root_dir_url}/child/f.mjsx"),
] ];
.iter()
.map(|f| ModuleSpecifier::parse(f).unwrap())
.collect::<Vec<_>>();
assert_eq!(result, expected); assert_eq!(
result
.into_iter()
.map(|s| s.to_string())
.collect::<Vec<_>>(),
expected
);
} }
#[tokio::test] #[tokio::test]

443
cli/util/glob.rs
View file

@ -5,30 +5,238 @@ use std::path::PathBuf;
use deno_core::anyhow::Context; use deno_core::anyhow::Context;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_core::normalize_path;
use deno_core::url::Url;
use indexmap::IndexMap;
pub fn expand_globs(paths: Vec<PathBuf>) -> Result<Vec<PathBuf>, AnyError> { use super::path::specifier_to_file_path;
let mut new_paths = vec![];
for path in paths {
let path_str = path.to_string_lossy();
if is_glob_pattern(&path_str) {
let globbed_paths = glob(&path_str)?;
for globbed_path_result in globbed_paths { #[derive(Clone, Default, Debug, Eq, PartialEq)]
new_paths.push(globbed_path_result?); pub struct FilePatterns {
pub include: Option<PathOrPatternSet>,
pub exclude: PathOrPatternSet,
}
impl FilePatterns {
pub fn matches_specifier(&self, specifier: &Url) -> bool {
let path = match specifier_to_file_path(specifier) {
Ok(path) => path,
Err(_) => return true,
};
self.matches_path(&path)
}
pub fn matches_path(&self, path: &Path) -> bool {
// Skip files in the exclude list.
if self.exclude.matches_path(path) {
return false;
}
// Ignore files not in the include list if it's present.
self
.include
.as_ref()
.map(|m| m.matches_path(path))
.unwrap_or(true)
}
/// Creates a collection of `FilePatterns` by base where the containing patterns
/// are only the ones applicable to the base.
///
/// The order these are returned in is the order that the directory traversal
/// should occur in.
pub fn split_by_base(&self) -> Vec<(PathBuf, Self)> {
let Some(include) = &self.include else {
return Vec::new();
};
let mut include_paths = Vec::new();
let mut include_patterns = Vec::new();
for path_or_pattern in &include.0 {
match path_or_pattern {
PathOrPattern::Path(path) => include_paths.push((path.is_file(), path)),
PathOrPattern::Pattern(pattern) => include_patterns.push(pattern),
} }
} else { }
new_paths.push(path); let include_patterns_by_base_path = include_patterns.into_iter().fold(
IndexMap::new(),
|mut map: IndexMap<_, Vec<_>>, p| {
map.entry(p.base_path()).or_default().push(p);
map
},
);
let exclude_by_base_path = self
.exclude
.0
.iter()
.map(|s| (s.base_path(), s))
.collect::<Vec<_>>();
let get_applicable_excludes =
|is_file_path: bool, base_path: &PathBuf| -> Vec<PathOrPattern> {
exclude_by_base_path
.iter()
.filter_map(|(exclude_base_path, exclude)| {
match exclude {
PathOrPattern::Path(exclude_path) => {
// For explicitly specified files, ignore when the exclude path starts
// with it. Regardless, include excludes that are on a sub path of the dir.
if is_file_path && base_path.starts_with(exclude_path)
|| exclude_path.starts_with(base_path)
{
Some((*exclude).clone())
} else {
None
}
}
PathOrPattern::Pattern(_) => {
// include globs that's are sub paths or a parent path
if exclude_base_path.starts_with(base_path)
|| base_path.starts_with(exclude_base_path)
{
Some((*exclude).clone())
} else {
None
}
}
}
})
.collect::<Vec<_>>()
};
let mut result = Vec::with_capacity(
include_paths.len() + include_patterns_by_base_path.len(),
);
for (is_file, path) in include_paths {
let applicable_excludes = get_applicable_excludes(is_file, path);
result.push((
path.clone(),
Self {
include: Some(PathOrPatternSet::new(vec![PathOrPattern::Path(
path.clone(),
)])),
exclude: PathOrPatternSet::new(applicable_excludes),
},
));
}
// todo(dsherret): This could be further optimized by not including
// patterns that will only ever match another base.
for base_path in include_patterns_by_base_path.keys() {
let applicable_excludes = get_applicable_excludes(false, base_path);
let mut applicable_includes = Vec::new();
// get all patterns that apply to the current or ancestor directories
for path in base_path.ancestors() {
if let Some(patterns) = include_patterns_by_base_path.get(path) {
applicable_includes.extend(
patterns
.iter()
.map(|p| PathOrPattern::Pattern((*p).clone())),
);
}
}
result.push((
base_path.clone(),
Self {
include: Some(PathOrPatternSet::new(applicable_includes)),
exclude: PathOrPatternSet::new(applicable_excludes),
},
));
}
// Sort by the longest base path first. This ensures that we visit opted into
// nested directories first before visiting the parent directory. The directory
// traverser will handle not going into directories it's already been in.
result.sort_by(|a, b| b.0.as_os_str().len().cmp(&a.0.as_os_str().len()));
result
}
}
#[derive(Clone, Default, Debug, Eq, PartialEq)]
pub struct PathOrPatternSet(Vec<PathOrPattern>);
impl PathOrPatternSet {
pub fn new(elements: Vec<PathOrPattern>) -> Self {
Self(elements)
}
pub fn from_absolute_paths(path: Vec<PathBuf>) -> Result<Self, AnyError> {
Ok(Self(
path
.into_iter()
.map(PathOrPattern::new)
.collect::<Result<Vec<_>, _>>()?,
))
}
pub fn inner(&self) -> &Vec<PathOrPattern> {
&self.0
}
pub fn into_path_or_patterns(self) -> Vec<PathOrPattern> {
self.0
}
pub fn matches_path(&self, path: &Path) -> bool {
self.0.iter().any(|p| p.matches_path(path))
}
pub fn base_paths(&self) -> Vec<PathBuf> {
let mut result = Vec::with_capacity(self.0.len());
for element in &self.0 {
match element {
PathOrPattern::Path(path) => {
result.push(path.to_path_buf());
}
PathOrPattern::Pattern(pattern) => {
result.push(pattern.base_path());
}
}
}
result
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum PathOrPattern {
Path(PathBuf),
Pattern(GlobPattern),
}
impl PathOrPattern {
pub fn new(path: PathBuf) -> Result<Self, AnyError> {
let path_str = path.to_string_lossy();
// todo(dsherret): don't store URLs in PathBufs
if path_str.starts_with("http:")
|| path_str.starts_with("https:")
|| path_str.starts_with("file:")
{
return Ok(Self::Path(path));
}
GlobPattern::new_if_pattern(&path_str).map(|maybe_pattern| {
maybe_pattern
.map(PathOrPattern::Pattern)
.unwrap_or_else(|| PathOrPattern::Path(normalize_path(path)))
})
}
pub fn matches_path(&self, path: &Path) -> bool {
match self {
PathOrPattern::Path(p) => path.starts_with(p),
PathOrPattern::Pattern(p) => p.matches_path(path),
} }
} }
Ok(new_paths) pub fn base_path(&self) -> PathBuf {
} match self {
PathOrPattern::Path(p) => p.clone(),
pub fn glob(pattern: &str) -> Result<glob::Paths, AnyError> { PathOrPattern::Pattern(p) => p.base_path(),
glob::glob_with(&escape_brackets(pattern), match_options()) }
.with_context(|| format!("Failed to expand glob: \"{}\"", pattern)) }
} }
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct GlobPattern(glob::Pattern); pub struct GlobPattern(glob::Pattern);
impl GlobPattern { impl GlobPattern {
@ -40,35 +248,38 @@ impl GlobPattern {
} }
pub fn new(pattern: &str) -> Result<Self, AnyError> { pub fn new(pattern: &str) -> Result<Self, AnyError> {
let pattern = glob::Pattern::new(pattern) let pattern =
.with_context(|| format!("Failed to expand glob: \"{}\"", pattern))?; glob::Pattern::new(&escape_brackets(pattern).replace('\\', "/"))
.with_context(|| format!("Failed to expand glob: \"{}\"", pattern))?;
Ok(Self(pattern)) Ok(Self(pattern))
} }
pub fn matches_path(&self, path: &Path) -> bool { pub fn matches_path(&self, path: &Path) -> bool {
self.0.matches_path(path) self.0.matches_path_with(path, match_options())
}
}
pub struct GlobSet(Vec<GlobPattern>);
impl GlobSet {
pub fn new(matchers: Vec<GlobPattern>) -> Self {
Self(matchers)
} }
pub fn matches_path(&self, path: &Path) -> bool { pub fn base_path(&self) -> PathBuf {
for pattern in &self.0 { let base_path = self
if pattern.matches_path(path) { .0
return true; .as_str()
} .split('/')
} .take_while(|c| !has_glob_chars(c))
false .collect::<Vec<_>>()
.join(std::path::MAIN_SEPARATOR_STR);
PathBuf::from(base_path)
} }
} }
pub fn is_glob_pattern(path: &str) -> bool { pub fn is_glob_pattern(path: &str) -> bool {
path.chars().any(|c| matches!(c, '*' | '?')) !path.starts_with("http:")
&& !path.starts_with("https:")
&& !path.starts_with("file:")
&& has_glob_chars(path)
}
fn has_glob_chars(pattern: &str) -> bool {
// we don't support [ and ]
pattern.chars().any(|c| matches!(c, '*' | '?'))
} }
fn escape_brackets(pattern: &str) -> String { fn escape_brackets(pattern: &str) -> String {
@ -92,17 +303,161 @@ fn match_options() -> glob::MatchOptions {
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use pretty_assertions::assert_eq;
use test_util::TempDir;
use super::*; use super::*;
#[test] // For easier comparisons in tests.
pub fn glob_set_matches_path() { #[derive(Debug, PartialEq, Eq)]
let glob_set = GlobSet::new(vec![ struct ComparableFilePatterns {
GlobPattern::new("foo/bar").unwrap(), include: Option<Vec<String>>,
GlobPattern::new("foo/baz").unwrap(), exclude: Vec<String>,
]); }
assert!(glob_set.matches_path(Path::new("foo/bar"))); impl ComparableFilePatterns {
assert!(glob_set.matches_path(Path::new("foo/baz"))); pub fn new(root: &Path, file_patterns: &FilePatterns) -> Self {
assert!(!glob_set.matches_path(Path::new("foo/qux"))); fn path_or_pattern_to_string(root: &Path, p: &PathOrPattern) -> String {
match p {
PathOrPattern::Path(p) => p
.strip_prefix(root)
.unwrap()
.to_string_lossy()
.replace('\\', "/"),
PathOrPattern::Pattern(p) => p
.0
.as_str()
.strip_prefix(&format!(
"{}/",
root.to_string_lossy().replace('\\', "/")
))
.unwrap()
.to_string(),
}
}
Self {
include: file_patterns.include.as_ref().map(|p| {
p.0
.iter()
.map(|p| path_or_pattern_to_string(root, p))
.collect()
}),
exclude: file_patterns
.exclude
.0
.iter()
.map(|p| path_or_pattern_to_string(root, p))
.collect(),
}
}
pub fn from_split(
root: &Path,
patterns_by_base: &[(PathBuf, FilePatterns)],
) -> Vec<(String, ComparableFilePatterns)> {
patterns_by_base
.iter()
.map(|(base_path, file_patterns)| {
(
base_path
.strip_prefix(root)
.unwrap()
.to_string_lossy()
.replace('\\', "/"),
ComparableFilePatterns::new(root, file_patterns),
)
})
.collect()
}
}
#[test]
fn should_split_globs_by_base_dir() {
let temp_dir = TempDir::new();
let patterns = FilePatterns {
include: Some(PathOrPatternSet::new(vec![
PathOrPattern::Pattern(
GlobPattern::new(&format!(
"{}/inner/**/*.ts",
temp_dir.path().to_string_lossy().replace('\\', "/")
))
.unwrap(),
),
PathOrPattern::Pattern(
GlobPattern::new(&format!(
"{}/inner/sub/deeper/**/*.js",
temp_dir.path().to_string_lossy().replace('\\', "/")
))
.unwrap(),
),
PathOrPattern::Pattern(
GlobPattern::new(&format!(
"{}/other/**/*.js",
temp_dir.path().to_string_lossy().replace('\\', "/")
))
.unwrap(),
),
PathOrPattern::Path(temp_dir.path().join("sub/file.ts").to_path_buf()),
])),
exclude: PathOrPatternSet::new(vec![
PathOrPattern::Pattern(
GlobPattern::new(&format!(
"{}/inner/other/**/*.ts",
temp_dir.path().to_string_lossy().replace('\\', "/")
))
.unwrap(),
),
PathOrPattern::Path(
temp_dir
.path()
.join("inner/sub/deeper/file.js")
.to_path_buf(),
),
]),
};
let split = ComparableFilePatterns::from_split(
temp_dir.path().as_path(),
&patterns.split_by_base(),
);
assert_eq!(
split,
vec![
(
"inner/sub/deeper".to_string(),
ComparableFilePatterns {
include: Some(vec![
"inner/sub/deeper/**/*.js".to_string(),
"inner/**/*.ts".to_string(),
]),
exclude: vec!["inner/sub/deeper/file.js".to_string()],
}
),
(
"sub/file.ts".to_string(),
ComparableFilePatterns {
include: Some(vec!["sub/file.ts".to_string()]),
exclude: vec![],
}
),
(
"inner".to_string(),
ComparableFilePatterns {
include: Some(vec!["inner/**/*.ts".to_string()]),
exclude: vec![
"inner/other/**/*.ts".to_string(),
"inner/sub/deeper/file.js".to_string(),
],
}
),
(
"other".to_string(),
ComparableFilePatterns {
include: Some(vec!["other/**/*.js".to_string()]),
exclude: vec![],
}
)
]
);
} }
} }