feat(compile): Enable multiple roots for a standalone module graph (#17663)

This change will enable dynamic imports and web workers to use modules
not reachable from the main module, by passing a list of extra side
module roots as options to `deno compile`. 

This can be done by specifying "--include" flag that accepts a file path or a
URL. This flag can be specified multiple times, to include several modules.
The modules specified with "--include" flag, will be added to the produced
"eszip".

cli/args/flags.rs

@@ -83,6 +83,7 @@ pub struct CompileFlags {
   pub output: Option<PathBuf>,
   pub args: Vec<String>,
   pub target: Option<String>,
+  pub include: Vec<String>,
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -908,6 +909,20 @@ fn compile_subcommand<'a>() -> Command<'a> {
   runtime_args(Command::new("compile"), true, false)
     .trailing_var_arg(true)
     .arg(script_arg().required(true))
+    .arg(
+      Arg::new("include")
+        .long("include")
+        .help("UNSTABLE: Additional module to include in the module graph")
+        .long_help(
+          "Includes an additional module in the compiled executable's module \
+    graph. Use this flag if a dynamically imported module or a web worker main \
+    module fails to load in the executable. This flag can be passed multiple \
+    times, to include multiple additional modules.",
+        )
+        .takes_value(true)
+        .multiple_occurrences(true)
+        .value_hint(ValueHint::FilePath),
+    )
     .arg(
       Arg::new("output")
         .long("output")
@@ -2486,12 +2501,17 @@ fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   let source_file = script[0].to_string();
   let output = matches.value_of("output").map(PathBuf::from);
   let target = matches.value_of("target").map(String::from);
+  let include = match matches.values_of("include") {
+    Some(f) => f.map(String::from).collect(),
+    None => vec![],
+  };
 
   flags.subcommand = DenoSubcommand::Compile(CompileFlags {
     source_file,
     output,
     args,
     target,
+    include,
   });
 }
 
@@ -6242,6 +6262,7 @@ mod tests {
           output: None,
           args: vec![],
           target: None,
+          include: vec![]
         }),
         type_check_mode: TypeCheckMode::Local,
         ..Flags::default()
@@ -6261,6 +6282,7 @@ mod tests {
           output: Some(PathBuf::from("colors")),
           args: svec!["foo", "bar"],
           target: None,
+          include: vec![]
         }),
         import_map_path: Some("import_map.json".to_string()),
         no_remote: true,

cli/graph_util.rs

@@ -150,7 +150,7 @@ pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) {
 }
 
 pub async fn create_graph_and_maybe_check(
-  root: ModuleSpecifier,
+  roots: Vec<ModuleSpecifier>,
   ps: &ProcState,
 ) -> Result<Arc<deno_graph::ModuleGraph>, AnyError> {
   let mut cache = cache::FetchCacher::new(
@@ -176,7 +176,7 @@ pub async fn create_graph_and_maybe_check(
   build_graph_with_npm_resolution(
     &mut graph,
     &ps.npm_resolver,
-    vec![root],
+    roots,
     &mut cache,
     deno_graph::BuildOptions {
       is_dynamic: false,

cli/tests/integration/compile_tests.rs

@@ -592,3 +592,37 @@ fn dynamic_import() {
   .unwrap();
   assert_eq!(String::from_utf8(output.stdout).unwrap(), expected);
 }
+
+#[test]
+fn dynamic_import_unanalyzable() {
+  let _guard = util::http_server();
+  let dir = TempDir::new();
+  let exe = if cfg!(windows) {
+    dir.path().join("dynamic_import_unanalyzable.exe")
+  } else {
+    dir.path().join("dynamic_import_unanalyzable")
+  };
+  let output = util::deno_cmd()
+    .current_dir(util::root_path())
+    .arg("compile")
+    .arg("--allow-read")
+    .arg("--include")
+    .arg(util::testdata_path().join("./compile/dynamic_imports/import1.ts"))
+    .arg("--output")
+    .arg(&exe)
+    .arg(
+      util::testdata_path()
+        .join("./compile/dynamic_imports/main_unanalyzable.ts"),
+    )
+    .output()
+    .unwrap();
+  assert!(output.status.success());
+
+  let output = Command::new(&exe).env("NO_COLOR", "").output().unwrap();
+  assert!(output.status.success());
+  let expected = std::fs::read_to_string(
+    util::testdata_path().join("./compile/dynamic_imports/main.out"),
+  )
+  .unwrap();
+  assert_eq!(String::from_utf8(output.stdout).unwrap(), expected);
+}

cli/tests/testdata/compile/dynamic_imports/import_path

@@ -0,0 +1 @@
+./import1.ts

cli/tests/testdata/compile/dynamic_imports/main.ts

@@ -3,4 +3,4 @@ console.log("Starting the main module");
 setTimeout(() => {
   console.log("Dynamic importing");
   import("./import1.ts").then(() => console.log("Dynamic import done."));
-}, 500);
+}, 0);

cli/tests/testdata/compile/dynamic_imports/main_unanalyzable.ts

@@ -0,0 +1,18 @@
+import { join } from "https://deno.land/std@0.178.0/path/mod.ts";
+
+console.log("Starting the main module");
+
+// We load the dynamic import path from the file system, to make sure any
+// improvements in static analysis can't defeat the purpose of this test, which
+// is to make sure the `--include` flag works to add non-analyzed imports to the
+// module graph.
+const IMPORT_PATH_FILE_PATH = join(
+  Deno.cwd(),
+  "tests/testdata/compile/dynamic_imports/import_path",
+);
+
+setTimeout(async () => {
+  console.log("Dynamic importing");
+  const importPath = (await Deno.readTextFile(IMPORT_PATH_FILE_PATH)).trim();
+  import(importPath).then(() => console.log("Dynamic import done."));
+}, 0);

cli/tools/bundle.rs

@@ -45,7 +45,8 @@ pub async fn bundle(
       log::debug!(">>>>> bundle START");
       let ps = ProcState::from_options(cli_options).await?;
       let graph =
-        create_graph_and_maybe_check(module_specifier.clone(), &ps).await?;
+        create_graph_and_maybe_check(vec![module_specifier.clone()], &ps)
+          .await?;
 
       let mut paths_to_watch: Vec<PathBuf> = graph
         .specifiers()

cli/tools/standalone.rs

@@ -41,6 +41,14 @@ pub async fn compile(
   let ps = ProcState::build(flags).await?;
   let module_specifier =
     resolve_url_or_path(&compile_flags.source_file, ps.options.initial_cwd())?;
+  let module_roots = {
+    let mut vec = Vec::with_capacity(compile_flags.include.len() + 1);
+    vec.push(module_specifier.clone());
+    for side_module in &compile_flags.include {
+      vec.push(resolve_url_or_path(side_module, ps.options.initial_cwd())?);
+    }
+    vec
+  };
   let deno_dir = &ps.dir;
 
   let output_path = resolve_compile_executable_output_path(
@@ -49,10 +57,9 @@ pub async fn compile(
   )
   .await?;
 
-  let graph = Arc::try_unwrap(
+  let graph =
-    create_graph_and_maybe_check(module_specifier.clone(), &ps).await?,
+    Arc::try_unwrap(create_graph_and_maybe_check(module_roots, &ps).await?)
-  )
+      .unwrap();
-  .unwrap();
 
   // at the moment, we don't support npm specifiers in deno_compile, so show an error
   error_for_any_npm_specifier(&graph)?;
@@ -351,6 +358,7 @@ mod test {
         output: Some(PathBuf::from("./file")),
         args: Vec::new(),
         target: Some("x86_64-unknown-linux-gnu".to_string()),
+        include: vec![],
       },
       &std::env::current_dir().unwrap(),
     )
@@ -371,6 +379,7 @@ mod test {
         output: Some(PathBuf::from("./file")),
         args: Vec::new(),
         target: Some("x86_64-pc-windows-msvc".to_string()),
+        include: vec![],
       },
       &std::env::current_dir().unwrap(),
     )