feat(compile): ability to embed local data files (#26934)

```
> deno compile --allow-read=. --include data-file.txt main.js
```

This only applies to files on the filesystem. For remote modules, that's
going to have to wait for `import ... from "./data.txt" with { "type":
"bytes" }` or whatever it will be.

cli/args/flags.rs

@@ -1909,10 +1909,10 @@ On the first invocation with deno will download the proper binary and cache it i
         Arg::new("include")
           .long("include")
           .help(
-            cstr!("Includes an additional module in the compiled executable's module graph.
+            cstr!("Includes an additional module or local data file in the compiled executable.
   <p(245)>Use this flag if a dynamically imported module or a web worker main module
-  fails to load in the executable. This flag can be passed multiple times,
-  to include multiple additional modules.</>",
+  fails to load in the executable or to embed a file in the executable. This flag can
+  be passed multiple times, to include multiple additional modules.</>",
           ))
           .action(ArgAction::Append)
           .value_hint(ValueHint::FilePath)

cli/factory.rs

@@ -884,6 +884,7 @@ impl CliFactory {
     let cli_options = self.cli_options()?;
     Ok(DenoCompileBinaryWriter::new(
       self.cjs_tracker()?,
+      self.cli_options()?,
       self.deno_dir()?,
       self.emitter()?,
       self.file_fetcher()?,

cli/standalone/binary.rs

@@ -201,7 +201,8 @@ fn write_binary_bytes(
   compile_flags: &CompileFlags,
 ) -> Result<(), AnyError> {
   let data_section_bytes =
-    serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?;
+    serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)
+      .context("Serializing binary data section.")?;
 
   let target = compile_flags.resolve_target();
   if target.contains("linux") {
@@ -364,6 +365,7 @@ pub fn extract_standalone(
 
 pub struct DenoCompileBinaryWriter<'a> {
   cjs_tracker: &'a CjsTracker,
+  cli_options: &'a CliOptions,
   deno_dir: &'a DenoDir,
   emitter: &'a Emitter,
   file_fetcher: &'a FileFetcher,
@@ -377,6 +379,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
   #[allow(clippy::too_many_arguments)]
   pub fn new(
     cjs_tracker: &'a CjsTracker,
+    cli_options: &'a CliOptions,
     deno_dir: &'a DenoDir,
     emitter: &'a Emitter,
     file_fetcher: &'a FileFetcher,
@@ -387,6 +390,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
   ) -> Self {
     Self {
       cjs_tracker,
+      cli_options,
       deno_dir,
       emitter,
       file_fetcher,
@@ -403,8 +407,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     graph: &ModuleGraph,
     root_dir_url: StandaloneRelativeFileBaseUrl<'_>,
     entrypoint: &ModuleSpecifier,
+    include_files: &[ModuleSpecifier],
     compile_flags: &CompileFlags,
-    cli_options: &CliOptions,
   ) -> Result<(), AnyError> {
     // Select base binary based on target
     let mut original_binary = self.get_base_binary(compile_flags).await?;
@@ -417,7 +421,8 @@ impl<'a> DenoCompileBinaryWriter<'a> {
           target,
         )
       }
-      set_windows_binary_to_gui(&mut original_binary)?;
+      set_windows_binary_to_gui(&mut original_binary)
+        .context("Setting windows binary to GUI.")?;
     }
     if compile_flags.icon.is_some() {
       let target = compile_flags.resolve_target();
@@ -435,7 +440,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
         graph,
         root_dir_url,
         entrypoint,
-        cli_options,
+        include_files,
         compile_flags,
       )
       .await
@@ -478,10 +483,14 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     if !binary_path.exists() {
       self
         .download_base_binary(&download_directory, &binary_path_suffix)
-        .await?;
+        .await
+        .context("Setting up base binary.")?;
     }
 
-    let archive_data = std::fs::read(binary_path)?;
+    let read_file = |path: &Path| -> Result<Vec<u8>, AnyError> {
+      std::fs::read(path).with_context(|| format!("Reading {}", path.display()))
+    };
+    let archive_data = read_file(&binary_path)?;
     let temp_dir = tempfile::TempDir::new()?;
     let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs {
       exe_name: "denort",
@@ -490,7 +499,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       is_windows: target.contains("windows"),
       dest_path: temp_dir.path(),
     })?;
-    let base_binary = std::fs::read(base_binary_path)?;
+    let base_binary = read_file(&base_binary_path)?;
     drop(temp_dir); // delete the temp dir
     Ok(base_binary)
   }
@@ -518,15 +527,19 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     let bytes = match maybe_bytes {
       Some(bytes) => bytes,
       None => {
-        log::info!("Download could not be found, aborting");
-        deno_runtime::exit(1);
+        bail!("Download could not be found, aborting");
       }
     };
 
-    std::fs::create_dir_all(output_directory)?;
+    let create_dir_all = |dir: &Path| {
+      std::fs::create_dir_all(dir)
+        .with_context(|| format!("Creating {}", dir.display()))
+    };
+    create_dir_all(output_directory)?;
     let output_path = output_directory.join(binary_path_suffix);
-    std::fs::create_dir_all(output_path.parent().unwrap())?;
-    tokio::fs::write(output_path, bytes).await?;
+    create_dir_all(output_path.parent().unwrap())?;
+    std::fs::write(&output_path, bytes)
+      .with_context(|| format!("Writing {}", output_path.display()))?;
     Ok(())
   }
 
@@ -540,73 +553,79 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     graph: &ModuleGraph,
     root_dir_url: StandaloneRelativeFileBaseUrl<'_>,
     entrypoint: &ModuleSpecifier,
-    cli_options: &CliOptions,
+    include_files: &[ModuleSpecifier],
     compile_flags: &CompileFlags,
   ) -> Result<(), AnyError> {
-    let ca_data = match cli_options.ca_data() {
+    let ca_data = match self.cli_options.ca_data() {
       Some(CaData::File(ca_file)) => Some(
-        std::fs::read(ca_file)
-          .with_context(|| format!("Reading: {ca_file}"))?,
+        std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?,
       ),
       Some(CaData::Bytes(bytes)) => Some(bytes.clone()),
       None => None,
     };
     let root_path = root_dir_url.inner().to_file_path().unwrap();
-    let (maybe_npm_vfs, node_modules, npm_snapshot) = match self
-      .npm_resolver
-      .as_inner()
-    {
-      InnerCliNpmResolverRef::Managed(managed) => {
-        let snapshot =
-          managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
-        if !snapshot.as_serialized().packages.is_empty() {
-          let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?;
+    let (maybe_npm_vfs, node_modules, npm_snapshot) =
+      match self.npm_resolver.as_inner() {
+        InnerCliNpmResolverRef::Managed(managed) => {
+          let snapshot =
+            managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
+          if !snapshot.as_serialized().packages.is_empty() {
+            let npm_vfs_builder = self
+              .build_npm_vfs(&root_path)
+              .context("Building npm vfs.")?;
+            (
+              Some(npm_vfs_builder),
+              Some(NodeModules::Managed {
+                node_modules_dir: self
+                  .npm_resolver
+                  .root_node_modules_path()
+                  .map(|path| {
+                    root_dir_url
+                      .specifier_key(
+                        &ModuleSpecifier::from_directory_path(path).unwrap(),
+                      )
+                      .into_owned()
+                  }),
+              }),
+              Some(snapshot),
+            )
+          } else {
+            (None, None, None)
+          }
+        }
+        InnerCliNpmResolverRef::Byonm(resolver) => {
+          let npm_vfs_builder = self.build_npm_vfs(&root_path)?;
           (
             Some(npm_vfs_builder),
-            Some(NodeModules::Managed {
-              node_modules_dir: self.npm_resolver.root_node_modules_path().map(
-                |path| {
+            Some(NodeModules::Byonm {
+              root_node_modules_dir: resolver.root_node_modules_path().map(
+                |node_modules_dir| {
                   root_dir_url
                     .specifier_key(
-                      &ModuleSpecifier::from_directory_path(path).unwrap(),
+                      &ModuleSpecifier::from_directory_path(node_modules_dir)
+                        .unwrap(),
                     )
                     .into_owned()
                 },
               ),
             }),
-            Some(snapshot),
+            None,
           )
-        } else {
-          (None, None, None)
         }
-      }
-      InnerCliNpmResolverRef::Byonm(resolver) => {
-        let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?;
-        (
-          Some(npm_vfs_builder),
-          Some(NodeModules::Byonm {
-            root_node_modules_dir: resolver.root_node_modules_path().map(
-              |node_modules_dir| {
-                root_dir_url
-                  .specifier_key(
-                    &ModuleSpecifier::from_directory_path(node_modules_dir)
-                      .unwrap(),
-                  )
-                  .into_owned()
-              },
-            ),
-          }),
-          None,
-        )
-      }
-    };
+      };
     let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs {
       npm_vfs
     } else {
       VfsBuilder::new(root_path.clone())?
     };
+    for include_file in include_files {
+      let path = deno_path_util::url_to_file_path(include_file)?;
+      vfs
+        .add_file_at_path(&path)
+        .with_context(|| format!("Including {}", path.display()))?;
+    }
     let mut remote_modules_store = RemoteModulesStoreBuilder::default();
-    let mut code_cache_key_hasher = if cli_options.code_cache_enabled() {
+    let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() {
       Some(FastInsecureHasher::new_deno_versioned())
     } else {
       None
@@ -672,7 +691,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
     }
     remote_modules_store.add_redirects(&graph.redirects);
 
-    let env_vars_from_env_file = match cli_options.env_file_name() {
+    let env_vars_from_env_file = match self.cli_options.env_file_name() {
       Some(env_filenames) => {
         let mut aggregated_env_vars = IndexMap::new();
         for env_filename in env_filenames.iter().rev() {
@@ -688,16 +707,17 @@ impl<'a> DenoCompileBinaryWriter<'a> {
 
     let metadata = Metadata {
       argv: compile_flags.args.clone(),
-      seed: cli_options.seed(),
+      seed: self.cli_options.seed(),
       code_cache_key: code_cache_key_hasher.map(|h| h.finish()),
-      location: cli_options.location_flag().clone(),
-      permissions: cli_options.permission_flags().clone(),
-      v8_flags: cli_options.v8_flags().clone(),
-      unsafely_ignore_certificate_errors: cli_options
+      location: self.cli_options.location_flag().clone(),
+      permissions: self.cli_options.permission_flags().clone(),
+      v8_flags: self.cli_options.v8_flags().clone(),
+      unsafely_ignore_certificate_errors: self
+        .cli_options
         .unsafely_ignore_certificate_errors()
         .clone(),
-      log_level: cli_options.log_level(),
-      ca_stores: cli_options.ca_stores().clone(),
+      log_level: self.cli_options.log_level(),
+      ca_stores: self.cli_options.ca_stores().clone(),
       ca_data,
       env_vars_from_env_file,
       entrypoint_key: root_dir_url.specifier_key(entrypoint).into_owned(),
@@ -740,11 +760,11 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       node_modules,
       unstable_config: UnstableConfig {
         legacy_flag_enabled: false,
-        bare_node_builtins: cli_options.unstable_bare_node_builtins(),
-        sloppy_imports: cli_options.unstable_sloppy_imports(),
-        features: cli_options.unstable_features(),
+        bare_node_builtins: self.cli_options.unstable_bare_node_builtins(),
+        sloppy_imports: self.cli_options.unstable_sloppy_imports(),
+        features: self.cli_options.unstable_features(),
       },
-      otel_config: cli_options.otel_config(),
+      otel_config: self.cli_options.otel_config(),
     };
 
     write_binary_bytes(
@@ -756,13 +776,10 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       vfs,
       compile_flags,
     )
+    .context("Writing binary bytes")
   }
 
-  fn build_npm_vfs(
-    &self,
-    root_path: &Path,
-    cli_options: &CliOptions,
-  ) -> Result<VfsBuilder, AnyError> {
+  fn build_npm_vfs(&self, root_path: &Path) -> Result<VfsBuilder, AnyError> {
     fn maybe_warn_different_system(system_info: &NpmSystemInfo) {
       if system_info != &NpmSystemInfo::default() {
         log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning"));
@@ -839,13 +856,18 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       InnerCliNpmResolverRef::Byonm(_) => {
         maybe_warn_different_system(&self.npm_system_info);
         let mut builder = VfsBuilder::new(root_path.to_path_buf())?;
-        for pkg_json in cli_options.workspace().package_jsons() {
+        for pkg_json in self.cli_options.workspace().package_jsons() {
           builder.add_file_at_path(&pkg_json.path)?;
         }
         // traverse and add all the node_modules directories in the workspace
         let mut pending_dirs = VecDeque::new();
         pending_dirs.push_back(
-          cli_options.workspace().root_dir().to_file_path().unwrap(),
+          self
+            .cli_options
+            .workspace()
+            .root_dir()
+            .to_file_path()
+            .unwrap(),
         );
         while let Some(pending_dir) = pending_dirs.pop_front() {
           let mut entries = fs::read_dir(&pending_dir)

cli/standalone/virtual_fs.rs

@@ -51,7 +51,8 @@ pub struct VfsBuilder {
 
 impl VfsBuilder {
   pub fn new(root_path: PathBuf) -> Result<Self, AnyError> {
-    let root_path = canonicalize_path(&root_path)?;
+    let root_path = canonicalize_path(&root_path)
+      .with_context(|| format!("Canonicalizing {}", root_path.display()))?;
     log::debug!("Building vfs with root '{}'", root_path.display());
     Ok(Self {
       root_dir: VirtualDirectory {

cli/tools/compile.rs

@@ -7,6 +7,7 @@ use crate::factory::CliFactory;
 use crate::http_util::HttpClientProvider;
 use crate::standalone::binary::StandaloneRelativeFileBaseUrl;
 use crate::standalone::is_standalone_binary;
+use deno_ast::MediaType;
 use deno_ast::ModuleSpecifier;
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
@@ -31,15 +32,12 @@ pub async fn compile(
   let module_graph_creator = factory.module_graph_creator().await?;
   let binary_writer = factory.create_compile_binary_writer().await?;
   let http_client = factory.http_client_provider();
-  let module_specifier = cli_options.resolve_main_module()?;
-  let module_roots = {
-    let mut vec = Vec::with_capacity(compile_flags.include.len() + 1);
-    vec.push(module_specifier.clone());
-    for side_module in &compile_flags.include {
-      vec.push(resolve_url_or_path(side_module, cli_options.initial_cwd())?);
-    }
-    vec
-  };
+  let entrypoint = cli_options.resolve_main_module()?;
+  let (module_roots, include_files) = get_module_roots_and_include_files(
+    entrypoint,
+    &compile_flags,
+    cli_options.initial_cwd(),
+  )?;
 
   // this is not supported, so show a warning about it, but don't error in order
   // to allow someone to still run `deno compile` when this is in a deno.json
@@ -82,18 +80,22 @@ pub async fn compile(
   check_warn_tsconfig(&ts_config_for_emit);
   let root_dir_url = resolve_root_dir_from_specifiers(
     cli_options.workspace().root_dir(),
-    graph.specifiers().map(|(s, _)| s).chain(
-      cli_options
-        .node_modules_dir_path()
-        .and_then(|p| ModuleSpecifier::from_directory_path(p).ok())
-        .iter(),
-    ),
+    graph
+      .specifiers()
+      .map(|(s, _)| s)
+      .chain(
+        cli_options
+          .node_modules_dir_path()
+          .and_then(|p| ModuleSpecifier::from_directory_path(p).ok())
+          .iter(),
+      )
+      .chain(include_files.iter()),
   );
   log::debug!("Binary root dir: {}", root_dir_url);
   log::info!(
     "{} {} to {}",
     colors::green("Compile"),
-    module_specifier.to_string(),
+    entrypoint,
     output_path.display(),
   );
   validate_output_path(&output_path)?;
@@ -118,9 +120,9 @@ pub async fn compile(
       file,
       &graph,
       StandaloneRelativeFileBaseUrl::from(&root_dir_url),
-      module_specifier,
+      entrypoint,
+      &include_files,
       &compile_flags,
-      cli_options,
     )
     .await
     .with_context(|| {
@@ -212,6 +214,48 @@ fn validate_output_path(output_path: &Path) -> Result<(), AnyError> {
   Ok(())
 }
 
+fn get_module_roots_and_include_files(
+  entrypoint: &ModuleSpecifier,
+  compile_flags: &CompileFlags,
+  initial_cwd: &Path,
+) -> Result<(Vec<ModuleSpecifier>, Vec<ModuleSpecifier>), AnyError> {
+  fn is_module_graph_module(url: &ModuleSpecifier) -> bool {
+    if url.scheme() != "file" {
+      return true;
+    }
+    let media_type = MediaType::from_specifier(url);
+    match media_type {
+      MediaType::JavaScript
+      | MediaType::Jsx
+      | MediaType::Mjs
+      | MediaType::Cjs
+      | MediaType::TypeScript
+      | MediaType::Mts
+      | MediaType::Cts
+      | MediaType::Dts
+      | MediaType::Dmts
+      | MediaType::Dcts
+      | MediaType::Tsx
+      | MediaType::Json
+      | MediaType::Wasm => true,
+      MediaType::Css | MediaType::SourceMap | MediaType::Unknown => false,
+    }
+  }
+
+  let mut module_roots = Vec::with_capacity(compile_flags.include.len() + 1);
+  let mut include_files = Vec::with_capacity(compile_flags.include.len());
+  module_roots.push(entrypoint.clone());
+  for side_module in &compile_flags.include {
+    let url = resolve_url_or_path(side_module, initial_cwd)?;
+    if is_module_graph_module(&url) {
+      module_roots.push(url);
+    } else {
+      include_files.push(url);
+    }
+  }
+  Ok((module_roots, include_files))
+}
+
 async fn resolve_compile_executable_output_path(
   http_client_provider: &HttpClientProvider,
   compile_flags: &CompileFlags,

tests/specs/compile/include_data_files/__test__.jsonc

@@ -0,0 +1,41 @@
+{
+  "tempDir": true,
+  "tests": {
+    "success": {
+      "steps": [{
+        "if": "unix",
+        "args": "compile --allow-read=data-file.txt --include data-file.txt --output main main.js",
+        "output": "[WILDCARD]"
+      }, {
+        "if": "unix",
+        "commandName": "./main",
+        "args": [],
+        "output": "output.out",
+        "exitCode": 0
+      }, {
+        "if": "windows",
+        "args": "compile --allow-read=data-file.txt --include data-file.txt --output main.exe main.js",
+        "output": "[WILDCARD]"
+      }, {
+        "if": "windows",
+        "commandName": "./main.exe",
+        "args": [],
+        "output": "output.out",
+        "exitCode": 0
+      }]
+    },
+    "non_existent": {
+      "steps": [{
+        "if": "unix",
+        "args": "compile --include does_not_exist.txt --output main main.js",
+        "output": "non_existent.out",
+        "exitCode": 1
+      }, {
+        "if": "windows",
+        "args": "compile --include does_not_exist.txt --output main.exe main.js",
+        "output": "non_existent.out",
+        "exitCode": 1
+      }]
+    }
+  }
+}

tests/specs/compile/include_data_files/data-file.txt

@@ -0,0 +1 @@
+Hi

tests/specs/compile/include_data_files/main.js

@@ -0,0 +1 @@
+console.log(Deno.readTextFileSync("./data-file.txt").trim());

tests/specs/compile/include_data_files/non_existent.out

@@ -0,0 +1,6 @@
+Compile file:///[WILDLINE]/main.js to [WILDLINE]
+error: Writing deno compile executable to temporary file 'main[WILDLINE]'
+
+Caused by:
+    0: Including [WILDLINE]does_not_exist.txt
+    1: [WILDLINE]

tests/specs/compile/include_data_files/output.out

@@ -0,0 +1 @@
+Hi