From b64ec7926831896f4e43b685891111409de45e85 Mon Sep 17 00:00:00 2001 From: Andreu Botella Date: Sun, 19 Mar 2023 00:43:07 +0100 Subject: [PATCH] feat(compile): Enable multiple roots for a standalone module graph (#17663) This change will enable dynamic imports and web workers to use modules not reachable from the main module, by passing a list of extra side module roots as options to `deno compile`. This can be done by specifying "--include" flag that accepts a file path or a URL. This flag can be specified multiple times, to include several modules. The modules specified with "--include" flag, will be added to the produced "eszip". --- cli/args/flags.rs | 22 ++++++++++++ cli/graph_util.rs | 4 +-- cli/tests/integration/compile_tests.rs | 34 +++++++++++++++++++ .../compile/dynamic_imports/import_path | 1 + .../testdata/compile/dynamic_imports/main.ts | 2 +- .../dynamic_imports/main_unanalyzable.ts | 18 ++++++++++ cli/tools/bundle.rs | 3 +- cli/tools/standalone.rs | 17 +++++++--- 8 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 cli/tests/testdata/compile/dynamic_imports/import_path create mode 100644 cli/tests/testdata/compile/dynamic_imports/main_unanalyzable.ts diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 0b1ba8c505..9650b9612e 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -83,6 +83,7 @@ pub struct CompileFlags { pub output: Option, pub args: Vec, pub target: Option, + pub include: Vec, } #[derive(Clone, Debug, Eq, PartialEq)] @@ -908,6 +909,20 @@ fn compile_subcommand<'a>() -> Command<'a> { runtime_args(Command::new("compile"), true, false) .trailing_var_arg(true) .arg(script_arg().required(true)) + .arg( + Arg::new("include") + .long("include") + .help("UNSTABLE: Additional module to include in the module graph") + .long_help( + "Includes an additional module in the compiled executable's module \ + graph. Use this flag if a dynamically imported module or a web worker main \ + module fails to load in the executable. This flag can be passed multiple \ + times, to include multiple additional modules.", + ) + .takes_value(true) + .multiple_occurrences(true) + .value_hint(ValueHint::FilePath), + ) .arg( Arg::new("output") .long("output") @@ -2486,12 +2501,17 @@ fn compile_parse(flags: &mut Flags, matches: &clap::ArgMatches) { let source_file = script[0].to_string(); let output = matches.value_of("output").map(PathBuf::from); let target = matches.value_of("target").map(String::from); + let include = match matches.values_of("include") { + Some(f) => f.map(String::from).collect(), + None => vec![], + }; flags.subcommand = DenoSubcommand::Compile(CompileFlags { source_file, output, args, target, + include, }); } @@ -6242,6 +6262,7 @@ mod tests { output: None, args: vec![], target: None, + include: vec![] }), type_check_mode: TypeCheckMode::Local, ..Flags::default() @@ -6261,6 +6282,7 @@ mod tests { output: Some(PathBuf::from("colors")), args: svec!["foo", "bar"], target: None, + include: vec![] }), import_map_path: Some("import_map.json".to_string()), no_remote: true, diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 0a94368339..d98d3cfd0c 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -150,7 +150,7 @@ pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) { } pub async fn create_graph_and_maybe_check( - root: ModuleSpecifier, + roots: Vec, ps: &ProcState, ) -> Result, AnyError> { let mut cache = cache::FetchCacher::new( @@ -176,7 +176,7 @@ pub async fn create_graph_and_maybe_check( build_graph_with_npm_resolution( &mut graph, &ps.npm_resolver, - vec![root], + roots, &mut cache, deno_graph::BuildOptions { is_dynamic: false, diff --git a/cli/tests/integration/compile_tests.rs b/cli/tests/integration/compile_tests.rs index add53434ff..828e04b1f0 100644 --- a/cli/tests/integration/compile_tests.rs +++ b/cli/tests/integration/compile_tests.rs @@ -592,3 +592,37 @@ fn dynamic_import() { .unwrap(); assert_eq!(String::from_utf8(output.stdout).unwrap(), expected); } + +#[test] +fn dynamic_import_unanalyzable() { + let _guard = util::http_server(); + let dir = TempDir::new(); + let exe = if cfg!(windows) { + dir.path().join("dynamic_import_unanalyzable.exe") + } else { + dir.path().join("dynamic_import_unanalyzable") + }; + let output = util::deno_cmd() + .current_dir(util::root_path()) + .arg("compile") + .arg("--allow-read") + .arg("--include") + .arg(util::testdata_path().join("./compile/dynamic_imports/import1.ts")) + .arg("--output") + .arg(&exe) + .arg( + util::testdata_path() + .join("./compile/dynamic_imports/main_unanalyzable.ts"), + ) + .output() + .unwrap(); + assert!(output.status.success()); + + let output = Command::new(&exe).env("NO_COLOR", "").output().unwrap(); + assert!(output.status.success()); + let expected = std::fs::read_to_string( + util::testdata_path().join("./compile/dynamic_imports/main.out"), + ) + .unwrap(); + assert_eq!(String::from_utf8(output.stdout).unwrap(), expected); +} diff --git a/cli/tests/testdata/compile/dynamic_imports/import_path b/cli/tests/testdata/compile/dynamic_imports/import_path new file mode 100644 index 0000000000..98222a2088 --- /dev/null +++ b/cli/tests/testdata/compile/dynamic_imports/import_path @@ -0,0 +1 @@ +./import1.ts diff --git a/cli/tests/testdata/compile/dynamic_imports/main.ts b/cli/tests/testdata/compile/dynamic_imports/main.ts index 7b4c487be5..b889e22039 100644 --- a/cli/tests/testdata/compile/dynamic_imports/main.ts +++ b/cli/tests/testdata/compile/dynamic_imports/main.ts @@ -3,4 +3,4 @@ console.log("Starting the main module"); setTimeout(() => { console.log("Dynamic importing"); import("./import1.ts").then(() => console.log("Dynamic import done.")); -}, 500); +}, 0); diff --git a/cli/tests/testdata/compile/dynamic_imports/main_unanalyzable.ts b/cli/tests/testdata/compile/dynamic_imports/main_unanalyzable.ts new file mode 100644 index 0000000000..e8e3e849b2 --- /dev/null +++ b/cli/tests/testdata/compile/dynamic_imports/main_unanalyzable.ts @@ -0,0 +1,18 @@ +import { join } from "https://deno.land/std@0.178.0/path/mod.ts"; + +console.log("Starting the main module"); + +// We load the dynamic import path from the file system, to make sure any +// improvements in static analysis can't defeat the purpose of this test, which +// is to make sure the `--include` flag works to add non-analyzed imports to the +// module graph. +const IMPORT_PATH_FILE_PATH = join( + Deno.cwd(), + "tests/testdata/compile/dynamic_imports/import_path", +); + +setTimeout(async () => { + console.log("Dynamic importing"); + const importPath = (await Deno.readTextFile(IMPORT_PATH_FILE_PATH)).trim(); + import(importPath).then(() => console.log("Dynamic import done.")); +}, 0); diff --git a/cli/tools/bundle.rs b/cli/tools/bundle.rs index 5a42f834ea..9420d9c8fb 100644 --- a/cli/tools/bundle.rs +++ b/cli/tools/bundle.rs @@ -45,7 +45,8 @@ pub async fn bundle( log::debug!(">>>>> bundle START"); let ps = ProcState::from_options(cli_options).await?; let graph = - create_graph_and_maybe_check(module_specifier.clone(), &ps).await?; + create_graph_and_maybe_check(vec![module_specifier.clone()], &ps) + .await?; let mut paths_to_watch: Vec = graph .specifiers() diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs index f0f53d417e..dcd2f5d437 100644 --- a/cli/tools/standalone.rs +++ b/cli/tools/standalone.rs @@ -41,6 +41,14 @@ pub async fn compile( let ps = ProcState::build(flags).await?; let module_specifier = resolve_url_or_path(&compile_flags.source_file, ps.options.initial_cwd())?; + let module_roots = { + let mut vec = Vec::with_capacity(compile_flags.include.len() + 1); + vec.push(module_specifier.clone()); + for side_module in &compile_flags.include { + vec.push(resolve_url_or_path(side_module, ps.options.initial_cwd())?); + } + vec + }; let deno_dir = &ps.dir; let output_path = resolve_compile_executable_output_path( @@ -49,10 +57,9 @@ pub async fn compile( ) .await?; - let graph = Arc::try_unwrap( - create_graph_and_maybe_check(module_specifier.clone(), &ps).await?, - ) - .unwrap(); + let graph = + Arc::try_unwrap(create_graph_and_maybe_check(module_roots, &ps).await?) + .unwrap(); // at the moment, we don't support npm specifiers in deno_compile, so show an error error_for_any_npm_specifier(&graph)?; @@ -351,6 +358,7 @@ mod test { output: Some(PathBuf::from("./file")), args: Vec::new(), target: Some("x86_64-unknown-linux-gnu".to_string()), + include: vec![], }, &std::env::current_dir().unwrap(), ) @@ -371,6 +379,7 @@ mod test { output: Some(PathBuf::from("./file")), args: Vec::new(), target: Some("x86_64-pc-windows-msvc".to_string()), + include: vec![], }, &std::env::current_dir().unwrap(), )