From 75945cbb86c25300a69ae1da47751affa3465929 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 16 Dec 2024 09:37:39 -0500 Subject: [PATCH] fix(compile): display embedded file sizes and total (#27360) Merging as a fix so that LTS gets this as it's a useful diagnostic tool. The 1MB unique is because we deduplicate files that we store (ex. some packages have the same file multiple times so we store that once). --- cli/standalone/virtual_fs.rs | 299 ++++++++++++++---- tests/integration/compile_tests.rs | 2 +- .../compile/env_vars_support/compile.out | 6 +- .../compile.out | 84 ++--- .../compile/include/symlink_twice/compile.out | 8 +- tests/specs/compile/npm_fs/compile.out | 8 +- .../compile/package_json_type/compile.out | 2 +- .../main_compile_file.out | 2 +- .../main_compile_folder.out | 14 +- 9 files changed, 299 insertions(+), 126 deletions(-) diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 8ddd179c7a..04e66d680e 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -35,6 +35,7 @@ use serde::Serialize; use thiserror::Error; use crate::util; +use crate::util::display::human_size; use crate::util::display::DisplayTreeNode; use crate::util::fs::canonicalize_path; @@ -512,96 +513,238 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) { let mut text = String::new(); let display_tree = vfs_as_display_tree(vfs, executable_name); display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string - log::info!( - "\n{}\n", - deno_terminal::colors::bold("Embedded File System") - ); + log::info!("\n{}\n", deno_terminal::colors::bold("Embedded Files")); log::info!("{}\n", text.trim()); + log::info!( + "Size: {}\n", + human_size(vfs.files.iter().map(|f| f.len() as f64).sum()) + ); } fn vfs_as_display_tree( vfs: &BuiltVfs, executable_name: &str, ) -> DisplayTreeNode { + /// The VFS only stores duplicate files once, so track that and display + /// it to the user so that it's not confusing. + #[derive(Debug, Default, Copy, Clone)] + struct Size { + unique: u64, + total: u64, + } + + impl std::ops::Add for Size { + type Output = Self; + + fn add(self, other: Self) -> Self { + Self { + unique: self.unique + other.unique, + total: self.total + other.total, + } + } + } + + impl std::iter::Sum for Size { + fn sum>(iter: I) -> Self { + iter.fold(Self::default(), std::ops::Add::add) + } + } + enum EntryOutput<'a> { - All, + All(Size), Subset(Vec>), - File, + File(Size), Symlink(&'a [String]), } impl<'a> EntryOutput<'a> { - pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { - let mut children = match self { - EntryOutput::Subset(vec) => vec - .iter() - .map(|e| e.output.as_display_tree(e.name.to_string())) - .collect(), - EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => { - vec![] + pub fn size(&self) -> Size { + match self { + EntryOutput::All(size) => *size, + EntryOutput::Subset(children) => { + children.iter().map(|c| c.output.size()).sum() } - }; - // we only want to collapse leafs so that nodes of the - // same depth have the same indentation - let collapse_single_child = - children.len() == 1 && children[0].children.is_empty(); + EntryOutput::File(size) => *size, + EntryOutput::Symlink(_) => Size { + unique: 0, + total: 0, + }, + } + } + } + + impl<'a> EntryOutput<'a> { + pub fn as_display_tree(&self, name: String) -> DisplayTreeNode { + fn format_size(size: Size) -> String { + if size.unique == size.total { + human_size(size.unique as f64) + } else { + format!( + "{}{}", + human_size(size.total as f64), + deno_terminal::colors::gray(format!( + " - {} unique", + human_size(size.unique as f64) + )) + ) + } + } + DisplayTreeNode { text: match self { - EntryOutput::All => format!("{}/*", name), - EntryOutput::Subset(_) => { - if collapse_single_child { - format!("{}/{}", name, children[0].text) - } else { - name - } + EntryOutput::All(size) => { + format!("{}/* ({})", name, format_size(*size)) + } + EntryOutput::Subset(children) => { + let size = children.iter().map(|c| c.output.size()).sum::(); + format!("{} ({})", name, format_size(size)) + } + EntryOutput::File(size) => { + format!("{} ({})", name, format_size(*size)) } - EntryOutput::File => name, EntryOutput::Symlink(parts) => { format!("{} --> {}", name, parts.join("/")) } }, - children: if collapse_single_child { - children.remove(0).children - } else { - children + children: match self { + EntryOutput::All(_) => Vec::new(), + EntryOutput::Subset(children) => children + .iter() + .map(|entry| entry.output.as_display_tree(entry.name.to_string())) + .collect(), + EntryOutput::File(_) => Vec::new(), + EntryOutput::Symlink(_) => Vec::new(), }, } } } pub struct DirEntryOutput<'a> { - name: &'a str, + name: Cow<'a, str>, output: EntryOutput<'a>, } - fn show_global_node_modules_dir( - vfs_dir: &VirtualDirectory, - ) -> Vec { - fn show_subset_deep( - vfs_dir: &VirtualDirectory, - depth: usize, - ) -> EntryOutput { - if depth == 0 { - EntryOutput::All + impl<'a> DirEntryOutput<'a> { + /// Collapses leaf nodes so they don't take up so much space when being + /// displayed. + /// + /// We only want to collapse leafs so that nodes of the same depth have + /// the same indentation. + pub fn collapse_leaf_nodes(&mut self) { + let EntryOutput::Subset(vec) = &mut self.output else { + return; + }; + for dir_entry in vec.iter_mut() { + dir_entry.collapse_leaf_nodes(); + } + if vec.len() != 1 { + return; + } + let child = &mut vec[0]; + let child_name = &child.name; + match &mut child.output { + EntryOutput::All(size) => { + self.name = Cow::Owned(format!("{}/{}", self.name, child_name)); + self.output = EntryOutput::All(*size); + } + EntryOutput::Subset(children) => { + if children.is_empty() { + self.name = Cow::Owned(format!("{}/{}", self.name, child_name)); + self.output = EntryOutput::Subset(vec![]); + } + } + EntryOutput::File(size) => { + self.name = Cow::Owned(format!("{}/{}", self.name, child_name)); + self.output = EntryOutput::File(*size); + } + EntryOutput::Symlink(parts) => { + let new_name = format!("{}/{}", self.name, child_name); + self.output = EntryOutput::Symlink(parts); + self.name = Cow::Owned(new_name); + } + } + } + } + + fn file_size(file: &VirtualFile, seen_offsets: &mut HashSet) -> Size { + fn add_offset_to_size( + offset: OffsetWithLength, + size: &mut Size, + seen_offsets: &mut HashSet, + ) { + if offset.len == 0 { + // some empty files have a dummy offset, so don't + // insert them into the seen offsets + return; + } + + if seen_offsets.insert(offset.offset) { + size.total += offset.len; + size.unique += offset.len; } else { - EntryOutput::Subset(show_subset(vfs_dir, depth)) + size.total += offset.len; } } - fn show_subset( - vfs_dir: &VirtualDirectory, + let mut size = Size::default(); + add_offset_to_size(file.offset, &mut size, seen_offsets); + if file.module_graph_offset.offset != file.offset.offset { + add_offset_to_size(file.module_graph_offset, &mut size, seen_offsets); + } + size + } + + fn dir_size(dir: &VirtualDirectory, seen_offsets: &mut HashSet) -> Size { + let mut size = Size::default(); + for entry in &dir.entries { + match entry { + VfsEntry::Dir(virtual_directory) => { + size = size + dir_size(virtual_directory, seen_offsets); + } + VfsEntry::File(file) => { + size = size + file_size(file, seen_offsets); + } + VfsEntry::Symlink(_) => { + // ignore + } + } + } + size + } + + fn show_global_node_modules_dir<'a>( + vfs_dir: &'a VirtualDirectory, + seen_offsets: &mut HashSet, + ) -> Vec> { + fn show_subset_deep<'a>( + vfs_dir: &'a VirtualDirectory, depth: usize, - ) -> Vec { + seen_offsets: &mut HashSet, + ) -> EntryOutput<'a> { + if depth == 0 { + EntryOutput::All(dir_size(vfs_dir, seen_offsets)) + } else { + EntryOutput::Subset(show_subset(vfs_dir, depth, seen_offsets)) + } + } + + fn show_subset<'a>( + vfs_dir: &'a VirtualDirectory, + depth: usize, + seen_offsets: &mut HashSet, + ) -> Vec> { vfs_dir .entries .iter() .map(|entry| DirEntryOutput { - name: entry.name(), + name: Cow::Borrowed(entry.name()), output: match entry { VfsEntry::Dir(virtual_directory) => { - show_subset_deep(virtual_directory, depth - 1) + show_subset_deep(virtual_directory, depth - 1, seen_offsets) + } + VfsEntry::File(file) => { + EntryOutput::File(file_size(file, seen_offsets)) } - VfsEntry::File(_) => EntryOutput::File, VfsEntry::Symlink(virtual_symlink) => { EntryOutput::Symlink(&virtual_symlink.dest_parts.0) } @@ -612,40 +755,54 @@ fn vfs_as_display_tree( // in this scenario, we want to show // .deno_compile_node_modules/localhost///* - show_subset(vfs_dir, 3) + show_subset(vfs_dir, 3, seen_offsets) } fn include_all_entries<'a>( dir_path: &WindowsSystemRootablePath, vfs_dir: &'a VirtualDirectory, + seen_offsets: &mut HashSet, ) -> Vec> { if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { - return show_global_node_modules_dir(vfs_dir); + return show_global_node_modules_dir(vfs_dir, seen_offsets); } vfs_dir .entries .iter() .map(|entry| DirEntryOutput { - name: entry.name(), - output: analyze_entry(dir_path.join(entry.name()), entry), + name: Cow::Borrowed(entry.name()), + output: analyze_entry(dir_path.join(entry.name()), entry, seen_offsets), }) .collect() } - fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput { + fn analyze_entry<'a>( + path: PathBuf, + entry: &'a VfsEntry, + seen_offsets: &mut HashSet, + ) -> EntryOutput<'a> { match entry { - VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory), - VfsEntry::File(_) => EntryOutput::File, + VfsEntry::Dir(virtual_directory) => { + analyze_dir(path, virtual_directory, seen_offsets) + } + VfsEntry::File(file) => EntryOutput::File(file_size(file, seen_offsets)), VfsEntry::Symlink(virtual_symlink) => { EntryOutput::Symlink(&virtual_symlink.dest_parts.0) } } } - fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput { + fn analyze_dir<'a>( + dir: PathBuf, + vfs_dir: &'a VirtualDirectory, + seen_offsets: &mut HashSet, + ) -> EntryOutput<'a> { if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME { - return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir)); + return EntryOutput::Subset(show_global_node_modules_dir( + vfs_dir, + seen_offsets, + )); } let real_entry_count = std::fs::read_dir(&dir) @@ -657,15 +814,15 @@ fn vfs_as_display_tree( .entries .iter() .map(|entry| DirEntryOutput { - name: entry.name(), - output: analyze_entry(dir.join(entry.name()), entry), + name: Cow::Borrowed(entry.name()), + output: analyze_entry(dir.join(entry.name()), entry, seen_offsets), }) .collect::>(); if children .iter() - .all(|c| !matches!(c.output, EntryOutput::Subset(_))) + .all(|c| !matches!(c.output, EntryOutput::Subset { .. })) { - EntryOutput::All + EntryOutput::All(children.iter().map(|c| c.output.size()).sum()) } else { EntryOutput::Subset(children) } @@ -673,13 +830,19 @@ fn vfs_as_display_tree( EntryOutput::Subset(include_all_entries( &WindowsSystemRootablePath::Path(dir), vfs_dir, + seen_offsets, )) } } // always include all the entries for the root directory, otherwise the // user might not have context about what's being shown - let child_entries = include_all_entries(&vfs.root_path, &vfs.root); + let mut seen_offsets = HashSet::with_capacity(vfs.files.len()); + let mut child_entries = + include_all_entries(&vfs.root_path, &vfs.root, &mut seen_offsets); + for child_entry in &mut child_entries { + child_entry.collapse_leaf_nodes(); + } DisplayTreeNode { text: deno_terminal::colors::italic(executable_name).to_string(), children: child_entries @@ -1637,8 +1800,8 @@ mod test { let temp_dir = TempDir::new(); temp_dir.write("root.txt", ""); temp_dir.create_dir_all("a"); - temp_dir.write("a/a.txt", ""); - temp_dir.write("a/b.txt", ""); + temp_dir.write("a/a.txt", "data"); + temp_dir.write("a/b.txt", "other data"); temp_dir.create_dir_all("b"); temp_dir.write("b/a.txt", ""); temp_dir.write("b/b.txt", ""); @@ -1666,10 +1829,10 @@ mod test { assert_eq!( strip_ansi_codes(&text), r#"executable -├── a/* -├── b/a.txt -└─┬ c - ├── a.txt +├── a/* (14B) +├── b/a.txt (0B) +└─┬ c (8B) + ├── a.txt (8B) └── b.txt --> c/a.txt "# ); diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs index a34d2cdd1d..a69e873ab4 100644 --- a/tests/integration/compile_tests.rs +++ b/tests/integration/compile_tests.rs @@ -1091,7 +1091,7 @@ Warning Failed resolving symlink. Ignoring. Path: [WILDCARD] Message: [WILDCARD]) -Embedded File System +Embedded Files [WILDCARD] diff --git a/tests/specs/compile/env_vars_support/compile.out b/tests/specs/compile/env_vars_support/compile.out index 02332ea0eb..cba114b1f7 100644 --- a/tests/specs/compile/env_vars_support/compile.out +++ b/tests/specs/compile/env_vars_support/compile.out @@ -3,8 +3,10 @@ Check [WILDCARD]main.ts Compile [WILDCARD]main.ts to out[WILDCARD] Warning Environment variables from the file "environment.env" were embedded in the generated executable file -Embedded File System +Embedded Files out[WILDLINE] -└── main.ts +└── main.ts ([WILDLINE]) + +Size: [WILDLINE] diff --git a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out index c29c878593..fa1dd2bf9f 100644 --- a/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out +++ b/tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out @@ -1,47 +1,49 @@ [WILDCARD] Compile file:///[WILDLINE]/main.ts to [WILDLINE] -Embedded File System +Embedded Files main[WILDLINE] -├─┬ .deno_compile_node_modules -│ └─┬ localhost -│ ├─┬ ansi-regex -│ │ ├── 3.0.1/* -│ │ └── 5.0.1/* -│ ├── ansi-styles/4.3.0/* -│ ├── camelcase/5.3.1/* -│ ├── cliui/6.0.0/* -│ ├── color-convert/2.0.1/* -│ ├── color-name/1.1.4/* -│ ├── cowsay/1.5.0/* -│ ├── decamelize/1.2.0/* -│ ├── emoji-regex/8.0.0/* -│ ├── find-up/4.1.0/* -│ ├── get-caller-file/2.0.5/* -│ ├── get-stdin/8.0.0/* -│ ├─┬ is-fullwidth-code-point -│ │ ├── 2.0.0/* -│ │ └── 3.0.0/* -│ ├── locate-path/5.0.0/* -│ ├── p-limit/2.3.0/* -│ ├── p-locate/4.1.0/* -│ ├── p-try/2.2.0/* -│ ├── path-exists/4.0.0/* -│ ├── require-directory/2.1.1/* -│ ├── require-main-filename/2.0.0/* -│ ├── set-blocking/2.0.0/* -│ ├─┬ string-width -│ │ ├── 2.1.1/* -│ │ └── 4.2.3/* -│ ├─┬ strip-ansi -│ │ ├── 4.0.0/* -│ │ └── 6.0.1/* -│ ├── strip-final-newline/2.0.0/* -│ ├── which-module/2.0.0/* -│ ├── wrap-ansi/6.2.0/* -│ ├── y18n/4.0.3/* -│ ├── yargs/15.4.1/* -│ └── yargs-parser/18.1.3/* -└── main.ts +├─┬ .deno_compile_node_modules ([WILDLINE]) +│ └─┬ localhost ([WILDLINE]) +│ ├─┬ ansi-regex ([WILDLINE]) +│ │ ├── 3.0.1/* ([WILDLINE]) +│ │ └── 5.0.1/* ([WILDLINE]) +│ ├── ansi-styles/4.3.0/* ([WILDLINE]) +│ ├── camelcase/5.3.1/* ([WILDLINE]) +│ ├── cliui/6.0.0/* ([WILDLINE]) +│ ├── color-convert/2.0.1/* ([WILDLINE]) +│ ├── color-name/1.1.4/* ([WILDLINE]) +│ ├── cowsay/1.5.0/* ([WILDLINE]) +│ ├── decamelize/1.2.0/* ([WILDLINE]) +│ ├── emoji-regex/8.0.0/* ([WILDLINE]) +│ ├── find-up/4.1.0/* ([WILDLINE]) +│ ├── get-caller-file/2.0.5/* ([WILDLINE]) +│ ├── get-stdin/8.0.0/* ([WILDLINE]) +│ ├─┬ is-fullwidth-code-point ([WILDLINE]) +│ │ ├── 2.0.0/* ([WILDLINE]) +│ │ └── 3.0.0/* ([WILDLINE]) +│ ├── locate-path/5.0.0/* ([WILDLINE]) +│ ├── p-limit/2.3.0/* ([WILDLINE]) +│ ├── p-locate/4.1.0/* ([WILDLINE]) +│ ├── p-try/2.2.0/* ([WILDLINE]) +│ ├── path-exists/4.0.0/* ([WILDLINE]) +│ ├── require-directory/2.1.1/* ([WILDLINE]) +│ ├── require-main-filename/2.0.0/* ([WILDLINE]) +│ ├── set-blocking/2.0.0/* ([WILDLINE]) +│ ├─┬ string-width ([WILDLINE]) +│ │ ├── 2.1.1/* ([WILDLINE]) +│ │ └── 4.2.3/* ([WILDLINE]) +│ ├─┬ strip-ansi ([WILDLINE]) +│ │ ├── 4.0.0/* ([WILDLINE]) +│ │ └── 6.0.1/* ([WILDLINE]) +│ ├── strip-final-newline/2.0.0/* ([WILDLINE]) +│ ├── which-module/2.0.0/* ([WILDLINE]) +│ ├── wrap-ansi/6.2.0/* ([WILDLINE]) +│ ├── y18n/4.0.3/* ([WILDLINE]) +│ ├── yargs/15.4.1/* ([WILDLINE]) +│ └── yargs-parser/18.1.3/* ([WILDLINE]) +└── main.ts ([WILDLINE]) + +Size: [WILDLINE] diff --git a/tests/specs/compile/include/symlink_twice/compile.out b/tests/specs/compile/include/symlink_twice/compile.out index c57eb9b2f1..6ae93bf1cb 100644 --- a/tests/specs/compile/include/symlink_twice/compile.out +++ b/tests/specs/compile/include/symlink_twice/compile.out @@ -1,9 +1,11 @@ Compile [WILDLINE] -Embedded File System +Embedded Files main[WILDLINE] -├── index.js +├── index.js ([WILDLINE]) ├── link.js --> index.js -└── setup.js +└── setup.js ([WILDLINE]) + +Size: [WILDLINE] diff --git a/tests/specs/compile/npm_fs/compile.out b/tests/specs/compile/npm_fs/compile.out index 4944146788..c2ecef4015 100644 --- a/tests/specs/compile/npm_fs/compile.out +++ b/tests/specs/compile/npm_fs/compile.out @@ -1,8 +1,10 @@ [WILDCARD] -Embedded File System +Embedded Files main[WILDLINE] -├── main.ts -└── node_modules/* +├── main.ts ([WILDLINE]) +└── node_modules/* ([WILDLINE]) + +Size: [WILDLINE] diff --git a/tests/specs/compile/package_json_type/compile.out b/tests/specs/compile/package_json_type/compile.out index 0370192809..c03e63b71a 100644 --- a/tests/specs/compile/package_json_type/compile.out +++ b/tests/specs/compile/package_json_type/compile.out @@ -1,6 +1,6 @@ Check file:///[WILDLINE]/main.js Compile file:///[WILDLINE] -Embedded File System +Embedded Files [WILDCARD] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out index 633c2cca62..a96f1f71e1 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out @@ -1,5 +1,5 @@ Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD] -Embedded File System +Embedded Files [WILDCARD] diff --git a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out index 61f0a2456a..538aaa414c 100644 --- a/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out +++ b/tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out @@ -4,12 +4,14 @@ Initialize @denotest/esm-basic@1.0.0 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE] -Embedded File System +Embedded Files bin[WILDLINE] -├─┬ compile -│ └─┬ node_modules_symlink_outside -│ ├── main.ts -│ └── node_modules/* -└── some_folder/* +├─┬ compile ([WILDLINE]) +│ └─┬ node_modules_symlink_outside ([WILDLINE]) +│ ├── main.ts ([WILDLINE]) +│ └── node_modules/* ([WILDLINE]) +└── some_folder/* ([WILDLINE]) + +Size: [WILDLINE]