feat: eszip loading

cli/args/flags.rs

@@ -110,6 +110,7 @@ pub struct CompileFlags {
   pub target: Option<String>,
   pub no_terminal: bool,
   pub include: Vec<String>,
+  pub eszip: bool,
 }
 
 impl CompileFlags {
@@ -587,6 +588,7 @@ pub struct Flags {
   pub code_cache_enabled: bool,
   pub permissions: PermissionFlags,
   pub allow_scripts: PackagesAllowedScripts,
+  pub eszip: bool,
 }
 
 #[derive(Clone, Debug, Eq, PartialEq, Default, Serialize, Deserialize)]
@@ -3327,6 +3329,7 @@ fn runtime_args(
     .arg(seed_arg())
     .arg(enable_testing_features_arg())
     .arg(strace_ops_arg())
+    .arg(eszip_arg())
 }
 
 fn inspect_args(app: Command) -> Command {
@@ -3445,6 +3448,14 @@ fn frozen_lockfile_arg() -> Arg {
     .help("Error out if lockfile is out of date")
 }
 
+fn eszip_arg() -> Arg {
+  Arg::new("eszip-internal-do-not-use")
+    .hide(true)
+    .long("eszip-internal-do-not-use")
+    .action(ArgAction::SetTrue)
+    .help("Run eszip")
+}
+
 /// Used for subcommands that operate on executable scripts only.
 /// `deno fmt` has its own `--ext` arg because its possible values differ.
 /// If --ext is not provided and the script doesn't have a file extension,
@@ -3885,6 +3896,7 @@ fn compile_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   let output = matches.remove_one::<String>("output");
   let target = matches.remove_one::<String>("target");
   let no_terminal = matches.get_flag("no-terminal");
+  let eszip = matches.get_flag("eszip-internal-do-not-use");
   let include = match matches.remove_many::<String>("include") {
     Some(f) => f.collect(),
     None => vec![],
@@ -3898,6 +3910,7 @@ fn compile_parse(flags: &mut Flags, matches: &mut ArgMatches) {
     target,
     no_terminal,
     include,
+    eszip,
   });
 }
 
@@ -4726,6 +4739,7 @@ fn runtime_args_parse(
   enable_testing_features_arg_parse(flags, matches);
   env_file_arg_parse(flags, matches);
   strace_ops_parse(flags, matches);
+  eszip_arg_parse(flags, matches);
 }
 
 fn inspect_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
@@ -4811,6 +4825,12 @@ fn frozen_lockfile_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   }
 }
 
+fn eszip_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
+  if matches.get_flag("eszip-internal-do-not-use") {
+    flags.eszip = true;
+  }
+}
+
 fn ext_arg_parse(flags: &mut Flags, matches: &mut ArgMatches) {
   flags.ext = matches.remove_one::<String>("ext");
 }
@@ -9262,7 +9282,8 @@ mod tests {
           args: vec![],
           target: None,
           no_terminal: false,
-          include: vec![]
+          include: vec![],
+          eszip: false,
         }),
         type_check_mode: TypeCheckMode::Local,
         ..Flags::default()
@@ -9284,7 +9305,8 @@ mod tests {
           args: svec!["foo", "bar", "-p", "8080"],
           target: None,
           no_terminal: true,
-          include: vec![]
+          include: vec![],
+          eszip: false,
         }),
         import_map_path: Some("import_map.json".to_string()),
         no_remote: true,

cli/main.rs

@@ -177,6 +177,8 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
     DenoSubcommand::Run(run_flags) => spawn_subcommand(async move {
       if run_flags.is_stdin() {
         tools::run::run_from_stdin(flags).await
+      } else if flags.eszip {
+        tools::run::run_eszip(flags).await
       } else {
         tools::run::run_script(WorkerExecutionMode::Run, flags, run_flags.watch).await
       }

cli/mainrt.rs

@@ -30,6 +30,8 @@ use deno_runtime::fmt_errors::format_js_error;
 use deno_runtime::tokio_util::create_and_run_current_thread_with_maybe_metrics;
 pub use deno_runtime::UNSTABLE_GRANULAR_FLAGS;
 use deno_terminal::colors;
+use standalone::binary::load_npm_vfs;
+use standalone::DenoCompileFileSystem;
 
 use std::borrow::Cow;
 use std::collections::HashMap;
@@ -91,7 +93,15 @@ fn main() {
         let (metadata, eszip) = future.await?;
         util::logger::init(metadata.log_level);
         load_env_vars(&metadata.env_vars_from_env_file);
-        let exit_code = standalone::run(eszip, metadata).await?;
+        let image_name =
+          current_exe_path.file_name().unwrap().to_string_lossy();
+        let exit_code = standalone::run(
+          eszip,
+          metadata,
+          current_exe_path.as_os_str().as_encoded_bytes(),
+          &image_name,
+        )
+        .await?;
         std::process::exit(exit_code);
       }
       Ok(None) => Ok(()),

cli/standalone/mod.rs

@@ -53,6 +53,7 @@ use deno_core::RequestedModuleType;
 use deno_core::ResolutionKind;
 use deno_npm::npm_rc::ResolvedNpmRc;
 use deno_runtime::deno_fs;
+use deno_runtime::deno_fs::FileSystem;
 use deno_runtime::deno_node::analyze::NodeCodeTranslator;
 use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::deno_node::NodeResolver;
@@ -65,9 +66,16 @@ use deno_runtime::WorkerLogLevel;
 use deno_semver::npm::NpmPackageReqReference;
 use eszip::EszipRelativeFileBaseUrl;
 use import_map::parse_from_json;
+use sha2::Digest;
+use sha2::Sha256;
 use std::borrow::Cow;
+use std::future::Future;
+use std::os::unix::ffi::OsStrExt;
+use std::path::Path;
+use std::path::PathBuf;
 use std::rc::Rc;
 use std::sync::Arc;
+use virtual_fs::FileBackedVfs;
 
 pub mod binary;
 mod file_system;
@@ -79,7 +87,7 @@ pub use binary::DenoCompileBinaryWriter;
 
 use self::binary::load_npm_vfs;
 use self::binary::Metadata;
-use self::file_system::DenoCompileFileSystem;
+pub use self::file_system::DenoCompileFileSystem;
 
 struct WorkspaceEszipModule {
   specifier: ModuleSpecifier,
@@ -423,10 +431,16 @@ impl RootCertStoreProvider for StandaloneRootCertStoreProvider {
 pub async fn run(
   mut eszip: eszip::EszipV2,
   metadata: Metadata,
+  image_path: &[u8],
+  image_name: &str,
 ) -> Result<i32, AnyError> {
-  let current_exe_path = std::env::current_exe().unwrap();
-  let current_exe_name =
-    current_exe_path.file_name().unwrap().to_string_lossy();
+  let image_path_hash = Sha256::digest(image_path);
+  let mut image_path_hash_buf = [0u8; 40];
+  let image_path_hash = &*faster_hex::hex_encode(
+    &image_path_hash[12..32],
+    &mut image_path_hash_buf,
+  )
+  .unwrap();
   let maybe_cwd = std::env::current_dir().ok();
   let deno_dir_provider = Arc::new(DenoDirProvider::new(None));
   let root_cert_store_provider = Arc::new(StandaloneRootCertStoreProvider {
@@ -441,8 +455,8 @@ pub async fn run(
   ));
   // use a dummy npm registry url
   let npm_registry_url = ModuleSpecifier::parse("https://localhost/").unwrap();
-  let root_path =
-    std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name));
+  let root_path = std::env::temp_dir()
+    .join(format!("deno-compile-{}-{}", image_name, image_path_hash));
   let root_dir_url =
     Arc::new(ModuleSpecifier::from_directory_path(&root_path).unwrap());
   let main_module = root_dir_url.join(&metadata.entrypoint_key).unwrap();

cli/tools/compile.rs

@@ -14,7 +14,10 @@ use deno_core::resolve_url_or_path;
 use deno_graph::GraphKind;
 use deno_terminal::colors;
 use eszip::EszipRelativeFileBaseUrl;
+use eszip::EszipV2;
 use rand::Rng;
+use std::io::Read;
+use std::io::Write;
 use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -126,19 +129,21 @@ pub async fn compile(
   let mut file = std::fs::File::create(&temp_path).with_context(|| {
     format!("Opening temporary file '{}'", temp_path.display())
   })?;
-  let write_result = binary_writer
-    .write_bin(
-      &mut file,
-      eszip,
-      root_dir_url,
-      &module_specifier,
-      &compile_flags,
-      cli_options,
-    )
-    .await
-    .with_context(|| {
-      format!("Writing temporary file '{}'", temp_path.display())
-    });
+  let write_result = if compile_flags.eszip {
+    file.write_all(&eszip.into_bytes()).map_err(AnyError::from)
+  } else {
+    binary_writer
+      .write_bin(
+        &mut file,
+        eszip,
+        root_dir_url,
+        &module_specifier,
+        &compile_flags,
+        cli_options,
+      )
+      .await
+  }
+  .with_context(|| format!("Writing temporary file '{}'", temp_path.display()));
   drop(file);
 
   // set it as executable
@@ -191,7 +196,7 @@ fn validate_output_path(output_path: &Path) -> Result<(), AnyError> {
 
     // Make sure we don't overwrite any file not created by Deno compiler because
     // this filename is chosen automatically in some cases.
-    if !is_standalone_binary(output_path) {
+    if !is_standalone_binary(output_path) && !is_eszip(output_path) {
       bail!(
         concat!(
           "Could not compile to file '{}' because the file already exists ",
@@ -345,6 +350,17 @@ fn resolve_root_dir_from_specifiers<'a>(
   ModuleSpecifier::parse(found_dir).unwrap()
 }
 
+pub fn is_eszip(path: &Path) -> bool {
+  let Ok(mut file) = std::fs::File::open(path) else {
+    return false;
+  };
+  let mut magic = [0u8; 8];
+  if file.read_exact(&mut magic).is_err() {
+    return false;
+  }
+  EszipV2::has_magic(&magic)
+}
+
 #[cfg(test)]
 mod test {
   pub use super::*;
@@ -361,6 +377,7 @@ mod test {
         target: Some("x86_64-unknown-linux-gnu".to_string()),
         no_terminal: false,
         include: vec![],
+        eszip: false,
       },
       &std::env::current_dir().unwrap(),
     )
@@ -385,6 +402,7 @@ mod test {
         target: Some("x86_64-pc-windows-msvc".to_string()),
         include: vec![],
         no_terminal: false,
+        eszip: false,
       },
       &std::env::current_dir().unwrap(),
     )

cli/tools/run/mod.rs

@@ -2,17 +2,27 @@
 
 use std::io::Read;
 
+use deno_config::workspace::PackageJsonDepResolution;
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
+use deno_core::futures::io::BufReader;
+use deno_core::futures::io::Cursor;
+use deno_core::unsync::spawn;
 use deno_runtime::deno_permissions::Permissions;
 use deno_runtime::deno_permissions::PermissionsContainer;
 use deno_runtime::WorkerExecutionMode;
+use eszip::EszipV2;
 
+use crate::args::CaData;
 use crate::args::EvalFlags;
 use crate::args::Flags;
 use crate::args::WatchFlagsWithPaths;
 use crate::factory::CliFactory;
 use crate::factory::CliFactoryBuilder;
 use crate::file_fetcher::File;
+use crate::standalone::binary::Metadata;
+use crate::standalone::binary::SerializedWorkspaceResolver;
 use crate::util;
 use crate::util::file_watcher::WatcherRestartMode;
 
@@ -40,17 +50,14 @@ To grant permissions, set them before the script argument. For example:
 
   // TODO(bartlomieju): actually I think it will also fail if there's an import
   // map specified and bare specifier is used on the command line
-  let factory = CliFactory::from_flags(flags)?;
+  let factory = CliFactory::from_flags(flags.clone())?;
   let deno_dir = factory.deno_dir()?;
   let http_client = factory.http_client_provider();
   let cli_options = factory.cli_options();
-
-  if cli_options.unstable_sloppy_imports() {
-    log::warn!(
-      "{} Sloppy imports are not recommended and have a negative impact on performance.",
-      crate::colors::yellow("Warning"),
-    );
-  }
+  let permissions = PermissionsContainer::new(Permissions::from_options(
+    &cli_options.permissions_options()?,
+  )?);
+  let main_module = cli_options.resolve_main_module()?;
 
   // Run a background task that checks for available upgrades or output
   // if an earlier run of this background task found a new version of Deno.
@@ -60,13 +67,15 @@ To grant permissions, set them before the script argument. For example:
     deno_dir.upgrade_check_file_path(),
   );
 
-  let main_module = cli_options.resolve_main_module()?;
+  if cli_options.unstable_sloppy_imports() {
+    log::warn!(
+      "{} Sloppy imports are not recommended and have a negative impact on performance.",
+      crate::colors::yellow("Warning"),
+    );
+  }
 
   maybe_npm_install(&factory).await?;
 
-  let permissions = PermissionsContainer::new(Permissions::from_options(
-    &cli_options.permissions_options()?,
-  )?);
   let worker_factory = factory.create_cli_main_worker_factory().await?;
   let mut worker = worker_factory
     .create_main_worker(mode, main_module, permissions)
@@ -202,3 +211,65 @@ async fn maybe_npm_install(factory: &CliFactory) -> Result<(), AnyError> {
   }
   Ok(())
 }
+
+pub async fn run_eszip(flags: Flags) -> Result<i32, AnyError> {
+  // TODO(bartlomieju): actually I think it will also fail if there's an import
+  // map specified and bare specifier is used on the command line
+  let factory = CliFactory::from_flags(flags.clone())?;
+  let cli_options = factory.cli_options();
+  let file_fetcher = factory.file_fetcher()?;
+  let permissions = PermissionsContainer::new(Permissions::from_options(
+    &cli_options.permissions_options()?,
+  )?);
+  let main_module = cli_options.resolve_main_module()?;
+
+  // TODO: streaming load
+  let eszip = file_fetcher.fetch(&main_module, &permissions).await?;
+  let eszip = BufReader::new(Cursor::new(eszip.source));
+  let (eszip, loader) = EszipV2::parse(eszip).await?;
+  spawn(async move {
+    if let Err(e) = loader.await {
+      log::error!("Error loading ESZip: {}", e);
+      std::process::exit(1);
+    }
+  });
+  let ca_data = match cli_options.ca_data() {
+    Some(CaData::File(ca_file)) => Some(
+      std::fs::read(ca_file).with_context(|| format!("Reading: {ca_file}"))?,
+    ),
+    Some(CaData::Bytes(bytes)) => Some(bytes.clone()),
+    None => None,
+  };
+  let Some(entrypoint_key) = eszip.specifiers().into_iter().next() else {
+    bail!("No modules found in eszip");
+  };
+
+  crate::standalone::run(
+    eszip,
+    Metadata {
+      argv: flags.argv,
+      seed: flags.seed,
+      permissions: flags.permissions,
+      location: flags.location,
+      v8_flags: flags.v8_flags,
+      log_level: flags.log_level,
+      ca_stores: flags.ca_stores,
+      ca_data,
+      unsafely_ignore_certificate_errors: flags
+        .unsafely_ignore_certificate_errors,
+      entrypoint_key,
+      node_modules: None,
+      disable_deprecated_api_warning: false,
+      unstable_config: flags.unstable_config,
+      env_vars_from_env_file: Default::default(),
+      workspace_resolver: SerializedWorkspaceResolver {
+        import_map: None,
+        package_jsons: Default::default(),
+        pkg_json_resolution: PackageJsonDepResolution::Disabled,
+      },
+    },
+    main_module.to_string().as_bytes(),
+    "run-eszip",
+  )
+  .await
+}

tests/integration/compile_tests.rs

@@ -1285,3 +1285,32 @@ fn standalone_jsr_dynamic_import() {
   output.assert_exit_code(0);
   output.assert_matches_text("Hello world\n");
 }
+
+#[test]
+fn eszip_output() {
+  let context = TestContextBuilder::new().build();
+  let dir = context.temp_dir();
+  let eszip = dir.path().join("welcome.eszip");
+  let output = context
+    .new_command()
+    .args_vec([
+      "compile",
+      "--eszip-internal-do-not-use",
+      "--output",
+      &eszip.to_string_lossy(),
+      "../../tests/testdata/welcome.ts",
+    ])
+    .run();
+  output.assert_exit_code(0);
+  output.skip_output_check();
+  let output = context
+    .new_command()
+    .args_vec([
+      "run",
+      "--allow-read",
+      "--eszip-internal-do-not-use",
+      &eszip.to_string_lossy(),
+    ])
+    .run();
+  output.assert_matches_text("Welcome to Deno!\n");
+}