fix(compile): display embedded file sizes and total (#27360)

Merging as a fix so that LTS gets this as it's a useful diagnostic tool.

The 1MB unique is because we deduplicate files that we store (ex. some
packages have the same file multiple times so we store that once).

cli/standalone/virtual_fs.rs

@@ -35,6 +35,7 @@ use serde::Serialize;
 use thiserror::Error;
 
 use crate::util;
+use crate::util::display::human_size;
 use crate::util::display::DisplayTreeNode;
 use crate::util::fs::canonicalize_path;
 
@@ -512,96 +513,238 @@ pub fn output_vfs(vfs: &BuiltVfs, executable_name: &str) {
   let mut text = String::new();
   let display_tree = vfs_as_display_tree(vfs, executable_name);
   display_tree.print(&mut text).unwrap(); // unwrap ok because it's writing to a string
-  log::info!(
+  log::info!("\n{}\n", deno_terminal::colors::bold("Embedded Files"));
-    "\n{}\n",
-    deno_terminal::colors::bold("Embedded File System")
-  );
   log::info!("{}\n", text.trim());
+  log::info!(
+    "Size: {}\n",
+    human_size(vfs.files.iter().map(|f| f.len() as f64).sum())
+  );
 }
 
 fn vfs_as_display_tree(
   vfs: &BuiltVfs,
   executable_name: &str,
 ) -> DisplayTreeNode {
+  /// The VFS only stores duplicate files once, so track that and display
+  /// it to the user so that it's not confusing.
+  #[derive(Debug, Default, Copy, Clone)]
+  struct Size {
+    unique: u64,
+    total: u64,
+  }
+
+  impl std::ops::Add for Size {
+    type Output = Self;
+
+    fn add(self, other: Self) -> Self {
+      Self {
+        unique: self.unique + other.unique,
+        total: self.total + other.total,
+      }
+    }
+  }
+
+  impl std::iter::Sum for Size {
+    fn sum<I: Iterator<Item = Self>>(iter: I) -> Self {
+      iter.fold(Self::default(), std::ops::Add::add)
+    }
+  }
+
   enum EntryOutput<'a> {
-    All,
+    All(Size),
     Subset(Vec<DirEntryOutput<'a>>),
-    File,
+    File(Size),
     Symlink(&'a [String]),
   }
 
   impl<'a> EntryOutput<'a> {
-    pub fn as_display_tree(&self, name: String) -> DisplayTreeNode {
+    pub fn size(&self) -> Size {
-      let mut children = match self {
+      match self {
-        EntryOutput::Subset(vec) => vec
+        EntryOutput::All(size) => *size,
-          .iter()
+        EntryOutput::Subset(children) => {
-          .map(|e| e.output.as_display_tree(e.name.to_string()))
+          children.iter().map(|c| c.output.size()).sum()
-          .collect(),
-        EntryOutput::All | EntryOutput::File | EntryOutput::Symlink(_) => {
-          vec![]
         }
-      };
+        EntryOutput::File(size) => *size,
-      // we only want to collapse leafs so that nodes of the
+        EntryOutput::Symlink(_) => Size {
-      // same depth have the same indentation
+          unique: 0,
-      let collapse_single_child =
+          total: 0,
-        children.len() == 1 && children[0].children.is_empty();
+        },
+      }
+    }
+  }
+
+  impl<'a> EntryOutput<'a> {
+    pub fn as_display_tree(&self, name: String) -> DisplayTreeNode {
+      fn format_size(size: Size) -> String {
+        if size.unique == size.total {
+          human_size(size.unique as f64)
+        } else {
+          format!(
+            "{}{}",
+            human_size(size.total as f64),
+            deno_terminal::colors::gray(format!(
+              " - {} unique",
+              human_size(size.unique as f64)
+            ))
+          )
+        }
+      }
+
       DisplayTreeNode {
         text: match self {
-          EntryOutput::All => format!("{}/*", name),
+          EntryOutput::All(size) => {
-          EntryOutput::Subset(_) => {
+            format!("{}/* ({})", name, format_size(*size))
-            if collapse_single_child {
+          }
-              format!("{}/{}", name, children[0].text)
+          EntryOutput::Subset(children) => {
-            } else {
+            let size = children.iter().map(|c| c.output.size()).sum::<Size>();
-              name
+            format!("{} ({})", name, format_size(size))
-            }
+          }
+          EntryOutput::File(size) => {
+            format!("{} ({})", name, format_size(*size))
           }
-          EntryOutput::File => name,
           EntryOutput::Symlink(parts) => {
             format!("{} --> {}", name, parts.join("/"))
           }
         },
-        children: if collapse_single_child {
+        children: match self {
-          children.remove(0).children
+          EntryOutput::All(_) => Vec::new(),
-        } else {
+          EntryOutput::Subset(children) => children
-          children
+            .iter()
+            .map(|entry| entry.output.as_display_tree(entry.name.to_string()))
+            .collect(),
+          EntryOutput::File(_) => Vec::new(),
+          EntryOutput::Symlink(_) => Vec::new(),
         },
       }
     }
   }
 
   pub struct DirEntryOutput<'a> {
-    name: &'a str,
+    name: Cow<'a, str>,
     output: EntryOutput<'a>,
   }
 
-  fn show_global_node_modules_dir(
+  impl<'a> DirEntryOutput<'a> {
-    vfs_dir: &VirtualDirectory,
+    /// Collapses leaf nodes so they don't take up so much space when being
-  ) -> Vec<DirEntryOutput> {
+    /// displayed.
-    fn show_subset_deep(
+    ///
-      vfs_dir: &VirtualDirectory,
+    /// We only want to collapse leafs so that nodes of the same depth have
-      depth: usize,
+    /// the same indentation.
-    ) -> EntryOutput {
+    pub fn collapse_leaf_nodes(&mut self) {
-      if depth == 0 {
+      let EntryOutput::Subset(vec) = &mut self.output else {
-        EntryOutput::All
+        return;
+      };
+      for dir_entry in vec.iter_mut() {
+        dir_entry.collapse_leaf_nodes();
+      }
+      if vec.len() != 1 {
+        return;
+      }
+      let child = &mut vec[0];
+      let child_name = &child.name;
+      match &mut child.output {
+        EntryOutput::All(size) => {
+          self.name = Cow::Owned(format!("{}/{}", self.name, child_name));
+          self.output = EntryOutput::All(*size);
+        }
+        EntryOutput::Subset(children) => {
+          if children.is_empty() {
+            self.name = Cow::Owned(format!("{}/{}", self.name, child_name));
+            self.output = EntryOutput::Subset(vec![]);
+          }
+        }
+        EntryOutput::File(size) => {
+          self.name = Cow::Owned(format!("{}/{}", self.name, child_name));
+          self.output = EntryOutput::File(*size);
+        }
+        EntryOutput::Symlink(parts) => {
+          let new_name = format!("{}/{}", self.name, child_name);
+          self.output = EntryOutput::Symlink(parts);
+          self.name = Cow::Owned(new_name);
+        }
+      }
+    }
+  }
+
+  fn file_size(file: &VirtualFile, seen_offsets: &mut HashSet<u64>) -> Size {
+    fn add_offset_to_size(
+      offset: OffsetWithLength,
+      size: &mut Size,
+      seen_offsets: &mut HashSet<u64>,
+    ) {
+      if offset.len == 0 {
+        // some empty files have a dummy offset, so don't
+        // insert them into the seen offsets
+        return;
+      }
+
+      if seen_offsets.insert(offset.offset) {
+        size.total += offset.len;
+        size.unique += offset.len;
       } else {
-        EntryOutput::Subset(show_subset(vfs_dir, depth))
+        size.total += offset.len;
       }
     }
 
-    fn show_subset(
+    let mut size = Size::default();
-      vfs_dir: &VirtualDirectory,
+    add_offset_to_size(file.offset, &mut size, seen_offsets);
+    if file.module_graph_offset.offset != file.offset.offset {
+      add_offset_to_size(file.module_graph_offset, &mut size, seen_offsets);
+    }
+    size
+  }
+
+  fn dir_size(dir: &VirtualDirectory, seen_offsets: &mut HashSet<u64>) -> Size {
+    let mut size = Size::default();
+    for entry in &dir.entries {
+      match entry {
+        VfsEntry::Dir(virtual_directory) => {
+          size = size + dir_size(virtual_directory, seen_offsets);
+        }
+        VfsEntry::File(file) => {
+          size = size + file_size(file, seen_offsets);
+        }
+        VfsEntry::Symlink(_) => {
+          // ignore
+        }
+      }
+    }
+    size
+  }
+
+  fn show_global_node_modules_dir<'a>(
+    vfs_dir: &'a VirtualDirectory,
+    seen_offsets: &mut HashSet<u64>,
+  ) -> Vec<DirEntryOutput<'a>> {
+    fn show_subset_deep<'a>(
+      vfs_dir: &'a VirtualDirectory,
       depth: usize,
-    ) -> Vec<DirEntryOutput> {
+      seen_offsets: &mut HashSet<u64>,
+    ) -> EntryOutput<'a> {
+      if depth == 0 {
+        EntryOutput::All(dir_size(vfs_dir, seen_offsets))
+      } else {
+        EntryOutput::Subset(show_subset(vfs_dir, depth, seen_offsets))
+      }
+    }
+
+    fn show_subset<'a>(
+      vfs_dir: &'a VirtualDirectory,
+      depth: usize,
+      seen_offsets: &mut HashSet<u64>,
+    ) -> Vec<DirEntryOutput<'a>> {
       vfs_dir
         .entries
         .iter()
         .map(|entry| DirEntryOutput {
-          name: entry.name(),
+          name: Cow::Borrowed(entry.name()),
           output: match entry {
             VfsEntry::Dir(virtual_directory) => {
-              show_subset_deep(virtual_directory, depth - 1)
+              show_subset_deep(virtual_directory, depth - 1, seen_offsets)
+            }
+            VfsEntry::File(file) => {
+              EntryOutput::File(file_size(file, seen_offsets))
             }
-            VfsEntry::File(_) => EntryOutput::File,
             VfsEntry::Symlink(virtual_symlink) => {
               EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
             }
@@ -612,40 +755,54 @@ fn vfs_as_display_tree(
 
     // in this scenario, we want to show
     // .deno_compile_node_modules/localhost/<package_name>/<version>/*
-    show_subset(vfs_dir, 3)
+    show_subset(vfs_dir, 3, seen_offsets)
   }
 
   fn include_all_entries<'a>(
     dir_path: &WindowsSystemRootablePath,
     vfs_dir: &'a VirtualDirectory,
+    seen_offsets: &mut HashSet<u64>,
   ) -> Vec<DirEntryOutput<'a>> {
     if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
-      return show_global_node_modules_dir(vfs_dir);
+      return show_global_node_modules_dir(vfs_dir, seen_offsets);
     }
 
     vfs_dir
       .entries
       .iter()
       .map(|entry| DirEntryOutput {
-        name: entry.name(),
+        name: Cow::Borrowed(entry.name()),
-        output: analyze_entry(dir_path.join(entry.name()), entry),
+        output: analyze_entry(dir_path.join(entry.name()), entry, seen_offsets),
       })
       .collect()
   }
 
-  fn analyze_entry(path: PathBuf, entry: &VfsEntry) -> EntryOutput {
+  fn analyze_entry<'a>(
+    path: PathBuf,
+    entry: &'a VfsEntry,
+    seen_offsets: &mut HashSet<u64>,
+  ) -> EntryOutput<'a> {
     match entry {
-      VfsEntry::Dir(virtual_directory) => analyze_dir(path, virtual_directory),
+      VfsEntry::Dir(virtual_directory) => {
-      VfsEntry::File(_) => EntryOutput::File,
+        analyze_dir(path, virtual_directory, seen_offsets)
+      }
+      VfsEntry::File(file) => EntryOutput::File(file_size(file, seen_offsets)),
       VfsEntry::Symlink(virtual_symlink) => {
         EntryOutput::Symlink(&virtual_symlink.dest_parts.0)
       }
     }
   }
 
-  fn analyze_dir(dir: PathBuf, vfs_dir: &VirtualDirectory) -> EntryOutput {
+  fn analyze_dir<'a>(
+    dir: PathBuf,
+    vfs_dir: &'a VirtualDirectory,
+    seen_offsets: &mut HashSet<u64>,
+  ) -> EntryOutput<'a> {
     if vfs_dir.name == DENO_COMPILE_GLOBAL_NODE_MODULES_DIR_NAME {
-      return EntryOutput::Subset(show_global_node_modules_dir(vfs_dir));
+      return EntryOutput::Subset(show_global_node_modules_dir(
+        vfs_dir,
+        seen_offsets,
+      ));
     }
 
     let real_entry_count = std::fs::read_dir(&dir)
@@ -657,15 +814,15 @@ fn vfs_as_display_tree(
         .entries
         .iter()
         .map(|entry| DirEntryOutput {
-          name: entry.name(),
+          name: Cow::Borrowed(entry.name()),
-          output: analyze_entry(dir.join(entry.name()), entry),
+          output: analyze_entry(dir.join(entry.name()), entry, seen_offsets),
         })
         .collect::<Vec<_>>();
       if children
         .iter()
-        .all(|c| !matches!(c.output, EntryOutput::Subset(_)))
+        .all(|c| !matches!(c.output, EntryOutput::Subset { .. }))
       {
-        EntryOutput::All
+        EntryOutput::All(children.iter().map(|c| c.output.size()).sum())
       } else {
         EntryOutput::Subset(children)
       }
@@ -673,13 +830,19 @@ fn vfs_as_display_tree(
       EntryOutput::Subset(include_all_entries(
         &WindowsSystemRootablePath::Path(dir),
         vfs_dir,
+        seen_offsets,
       ))
     }
   }
 
   // always include all the entries for the root directory, otherwise the
   // user might not have context about what's being shown
-  let child_entries = include_all_entries(&vfs.root_path, &vfs.root);
+  let mut seen_offsets = HashSet::with_capacity(vfs.files.len());
+  let mut child_entries =
+    include_all_entries(&vfs.root_path, &vfs.root, &mut seen_offsets);
+  for child_entry in &mut child_entries {
+    child_entry.collapse_leaf_nodes();
+  }
   DisplayTreeNode {
     text: deno_terminal::colors::italic(executable_name).to_string(),
     children: child_entries
@@ -1637,8 +1800,8 @@ mod test {
     let temp_dir = TempDir::new();
     temp_dir.write("root.txt", "");
     temp_dir.create_dir_all("a");
-    temp_dir.write("a/a.txt", "");
+    temp_dir.write("a/a.txt", "data");
-    temp_dir.write("a/b.txt", "");
+    temp_dir.write("a/b.txt", "other data");
     temp_dir.create_dir_all("b");
     temp_dir.write("b/a.txt", "");
     temp_dir.write("b/b.txt", "");
@@ -1666,10 +1829,10 @@ mod test {
     assert_eq!(
       strip_ansi_codes(&text),
       r#"executable
-├── a/*
+├── a/* (14B)
-├── b/a.txt
+├── b/a.txt (0B)
-└─┬ c
+└─┬ c (8B)
-  ├── a.txt
+  ├── a.txt (8B)
   └── b.txt --> c/a.txt
 "#
     );

tests/integration/compile_tests.rs

@@ -1091,7 +1091,7 @@ Warning Failed resolving symlink. Ignoring.
     Path: [WILDCARD]
     Message: [WILDCARD])
 
-Embedded File System
+Embedded Files
 
 [WILDCARD]
 

tests/specs/compile/env_vars_support/compile.out

@@ -3,8 +3,10 @@ Check [WILDCARD]main.ts
 Compile [WILDCARD]main.ts to out[WILDCARD]
 Warning Environment variables from the file "environment.env" were embedded in the generated executable file
 
-Embedded File System
+Embedded Files
 
 out[WILDLINE]
-└── main.ts
+└── main.ts ([WILDLINE])
+
+Size: [WILDLINE]
 

tests/specs/compile/global_npm_cache_implicit_read_permission/compile.out

@@ -1,47 +1,49 @@
 [WILDCARD]
 Compile file:///[WILDLINE]/main.ts to [WILDLINE]
 
-Embedded File System
+Embedded Files
 
 main[WILDLINE]
-├─┬ .deno_compile_node_modules
+├─┬ .deno_compile_node_modules ([WILDLINE])
-│ └─┬ localhost
+│ └─┬ localhost ([WILDLINE])
-│   ├─┬ ansi-regex
+│   ├─┬ ansi-regex ([WILDLINE])
-│   │ ├── 3.0.1/*
+│   │ ├── 3.0.1/* ([WILDLINE])
-│   │ └── 5.0.1/*
+│   │ └── 5.0.1/* ([WILDLINE])
-│   ├── ansi-styles/4.3.0/*
+│   ├── ansi-styles/4.3.0/* ([WILDLINE])
-│   ├── camelcase/5.3.1/*
+│   ├── camelcase/5.3.1/* ([WILDLINE])
-│   ├── cliui/6.0.0/*
+│   ├── cliui/6.0.0/* ([WILDLINE])
-│   ├── color-convert/2.0.1/*
+│   ├── color-convert/2.0.1/* ([WILDLINE])
-│   ├── color-name/1.1.4/*
+│   ├── color-name/1.1.4/* ([WILDLINE])
-│   ├── cowsay/1.5.0/*
+│   ├── cowsay/1.5.0/* ([WILDLINE])
-│   ├── decamelize/1.2.0/*
+│   ├── decamelize/1.2.0/* ([WILDLINE])
-│   ├── emoji-regex/8.0.0/*
+│   ├── emoji-regex/8.0.0/* ([WILDLINE])
-│   ├── find-up/4.1.0/*
+│   ├── find-up/4.1.0/* ([WILDLINE])
-│   ├── get-caller-file/2.0.5/*
+│   ├── get-caller-file/2.0.5/* ([WILDLINE])
-│   ├── get-stdin/8.0.0/*
+│   ├── get-stdin/8.0.0/* ([WILDLINE])
-│   ├─┬ is-fullwidth-code-point
+│   ├─┬ is-fullwidth-code-point ([WILDLINE])
-│   │ ├── 2.0.0/*
+│   │ ├── 2.0.0/* ([WILDLINE])
-│   │ └── 3.0.0/*
+│   │ └── 3.0.0/* ([WILDLINE])
-│   ├── locate-path/5.0.0/*
+│   ├── locate-path/5.0.0/* ([WILDLINE])
-│   ├── p-limit/2.3.0/*
+│   ├── p-limit/2.3.0/* ([WILDLINE])
-│   ├── p-locate/4.1.0/*
+│   ├── p-locate/4.1.0/* ([WILDLINE])
-│   ├── p-try/2.2.0/*
+│   ├── p-try/2.2.0/* ([WILDLINE])
-│   ├── path-exists/4.0.0/*
+│   ├── path-exists/4.0.0/* ([WILDLINE])
-│   ├── require-directory/2.1.1/*
+│   ├── require-directory/2.1.1/* ([WILDLINE])
-│   ├── require-main-filename/2.0.0/*
+│   ├── require-main-filename/2.0.0/* ([WILDLINE])
-│   ├── set-blocking/2.0.0/*
+│   ├── set-blocking/2.0.0/* ([WILDLINE])
-│   ├─┬ string-width
+│   ├─┬ string-width ([WILDLINE])
-│   │ ├── 2.1.1/*
+│   │ ├── 2.1.1/* ([WILDLINE])
-│   │ └── 4.2.3/*
+│   │ └── 4.2.3/* ([WILDLINE])
-│   ├─┬ strip-ansi
+│   ├─┬ strip-ansi ([WILDLINE])
-│   │ ├── 4.0.0/*
+│   │ ├── 4.0.0/* ([WILDLINE])
-│   │ └── 6.0.1/*
+│   │ └── 6.0.1/* ([WILDLINE])
-│   ├── strip-final-newline/2.0.0/*
+│   ├── strip-final-newline/2.0.0/* ([WILDLINE])
-│   ├── which-module/2.0.0/*
+│   ├── which-module/2.0.0/* ([WILDLINE])
-│   ├── wrap-ansi/6.2.0/*
+│   ├── wrap-ansi/6.2.0/* ([WILDLINE])
-│   ├── y18n/4.0.3/*
+│   ├── y18n/4.0.3/* ([WILDLINE])
-│   ├── yargs/15.4.1/*
+│   ├── yargs/15.4.1/* ([WILDLINE])
-│   └── yargs-parser/18.1.3/*
+│   └── yargs-parser/18.1.3/* ([WILDLINE])
-└── main.ts
+└── main.ts ([WILDLINE])
+
+Size: [WILDLINE]
 

tests/specs/compile/include/symlink_twice/compile.out

@@ -1,9 +1,11 @@
 Compile [WILDLINE]
 
-Embedded File System
+Embedded Files
 
 main[WILDLINE]
-├── index.js
+├── index.js ([WILDLINE])
 ├── link.js --> index.js
-└── setup.js
+└── setup.js ([WILDLINE])
+
+Size: [WILDLINE]
 

tests/specs/compile/npm_fs/compile.out

@@ -1,8 +1,10 @@
 [WILDCARD]
 
-Embedded File System
+Embedded Files
 
 main[WILDLINE]
-├── main.ts
+├── main.ts ([WILDLINE])
-└── node_modules/*
+└── node_modules/* ([WILDLINE])
+
+Size: [WILDLINE]
 

tests/specs/compile/package_json_type/compile.out

@@ -1,6 +1,6 @@
 Check file:///[WILDLINE]/main.js
 Compile file:///[WILDLINE]
 
-Embedded File System
+Embedded Files
 
 [WILDCARD]

tests/testdata/compile/node_modules_symlink_outside/main_compile_file.out

@@ -1,5 +1,5 @@
 Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDCARD]
 
-Embedded File System
+Embedded Files
 
 [WILDCARD]

tests/testdata/compile/node_modules_symlink_outside/main_compile_folder.out

@@ -4,12 +4,14 @@ Initialize @denotest/esm-basic@1.0.0
 Check file:///[WILDCARD]/node_modules_symlink_outside/main.ts
 Compile file:///[WILDCARD]/node_modules_symlink_outside/main.ts to [WILDLINE]
 
-Embedded File System
+Embedded Files
 
 bin[WILDLINE]
-├─┬ compile
+├─┬ compile ([WILDLINE])
-│ └─┬ node_modules_symlink_outside
+│ └─┬ node_modules_symlink_outside ([WILDLINE])
-│   ├── main.ts
+│   ├── main.ts ([WILDLINE])
-│   └── node_modules/*
+│   └── node_modules/* ([WILDLINE])
-└── some_folder/*
+└── some_folder/* ([WILDLINE])
+
+Size: [WILDLINE]