build: add "include_js_files_for_snapshotting" Cargo feature (#17826)

This allows to not include source code into the binary (because
it will already be included in the V8 snapshot).

Nothing changes for the embedders - everything should still build the
same.

This commit brings the binary size from 87Mb to 82Mb on M1.

Alternative to https://github.com/denoland/deno/pull/17820 and
https://github.com/denoland/deno/pull/17653

---------

Co-authored-by: Leo Kettmeir <crowlkats@toaxl.com>

cli/Cargo.toml

@@ -27,8 +27,8 @@ harness = false
 path = "./bench/lsp_bench_standalone.rs"
 
 [build-dependencies]
-deno_runtime = { workspace = true, features = ["snapshot_from_snapshot"] }
-deno_core.workspace = true
+deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] }
+deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 regex.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -43,13 +43,13 @@ winres.workspace = true
 
 [dependencies]
 deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] }
-deno_core.workspace = true
+deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
 deno_doc = "0.55.0"
 deno_emit = "0.15.0"
 deno_graph = "0.43.2"
 deno_lint = { version = "0.38.0", features = ["docs"] }
 deno_lockfile.workspace = true
-deno_runtime.workspace = true
+deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] }
 deno_task_shell = "0.8.1"
 napi_sym.workspace = true
 

cli/build.rs

@@ -360,8 +360,8 @@ fn create_cli_snapshot(snapshot_path: PathBuf) {
   );
   esm_files.push(ExtensionFileSource {
     specifier: "runtime/js/99_main.js".to_string(),
-    code: ExtensionFileSourceCode::IncludedInBinary(
-      deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS,
+    code: ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
+      std::path::PathBuf::from(deno_runtime::js::PATH_FOR_99_MAIN_JS),
     ),
   });
   let extensions_with_js = vec![Extension::builder("cli")

core/Cargo.toml

@@ -16,6 +16,7 @@ path = "lib.rs"
 [features]
 default = ["v8_use_custom_libcxx"]
 v8_use_custom_libcxx = ["v8/use_custom_libcxx"]
+include_js_files_for_snapshotting = []
 
 [dependencies]
 anyhow.workspace = true

core/extensions.rs

@@ -1,7 +1,9 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 use crate::OpState;
+use anyhow::Context as _;
 use anyhow::Error;
 use std::cell::RefCell;
+use std::path::PathBuf;
 use std::rc::Rc;
 use std::task::Context;
 use v8::fast_api::FastFunction;
@@ -13,8 +15,24 @@ pub enum ExtensionFileSourceCode {
   /// will result in two copies of the source code being included - one in the
   /// snapshot, the other the static string in the `Extension`.
   IncludedInBinary(&'static str),
-  // TODO(bartlomieju): add more variants that allow to read file from the disk,
-  // and not include it in the binary.
+
+  // Source code is loaded from a file on disk. It's meant to be used if the
+  // embedder is creating snapshots. Files will be loaded from the filesystem
+  // during the build time and they will only be present in the V8 snapshot.
+  LoadedFromFsDuringSnapshot(PathBuf),
+}
+
+impl ExtensionFileSourceCode {
+  pub fn load(&self) -> Result<String, Error> {
+    match self {
+      ExtensionFileSourceCode::IncludedInBinary(code) => Ok(code.to_string()),
+      ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(path) => {
+        let msg = format!("Failed to read \"{}\"", path.display());
+        let code = std::fs::read_to_string(path).context(msg)?;
+        Ok(code)
+      }
+    }
+  }
 }
 
 #[derive(Clone, Debug)]
@@ -299,6 +317,7 @@ impl ExtensionBuilder {
 /// - "internal:my_extension/js/01_hello.js"
 /// - "internal:my_extension/js/02_goodbye.js"
 /// ```
+#[cfg(not(feature = "include_js_files_for_snapshotting"))]
 #[macro_export]
 macro_rules! include_js_files {
   (dir $dir:literal, $($file:literal,)+) => {
@@ -323,3 +342,29 @@ macro_rules! include_js_files {
     ]
   };
 }
+
+#[cfg(feature = "include_js_files_for_snapshotting")]
+#[macro_export]
+macro_rules! include_js_files {
+  (dir $dir:literal, $($file:literal,)+) => {
+    vec![
+      $($crate::ExtensionFileSource {
+        specifier: concat!($dir, "/", $file).to_string(),
+        code: $crate::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
+          std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join($dir).join($file)
+        ),
+      },)+
+    ]
+  };
+
+  ($($file:literal,)+) => {
+    vec![
+      $($crate::ExtensionFileSource {
+        specifier: $file.to_string(),
+        code: $crate::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
+          std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join($file)
+        ),
+      },)+
+    ]
+  };
+}

core/modules.rs

@@ -3,7 +3,6 @@
 use crate::bindings;
 use crate::error::generic_error;
 use crate::extensions::ExtensionFileSource;
-use crate::extensions::ExtensionFileSourceCode;
 use crate::module_specifier::ModuleSpecifier;
 use crate::resolve_import;
 use crate::resolve_url;
@@ -403,10 +402,9 @@ impl ModuleLoader for InternalModuleLoader {
       let result = if let Some(load_callback) = &self.maybe_load_callback {
         load_callback(file_source)
       } else {
-        match file_source.code {
-          ExtensionFileSourceCode::IncludedInBinary(code) => {
-            Ok(code.to_string())
-          }
+        match file_source.code.load() {
+          Ok(code) => Ok(code),
+          Err(err) => return futures::future::err(err).boxed_local(),
         }
       };
 

core/runtime.rs

@@ -21,7 +21,6 @@ use crate::source_map::SourceMapCache;
 use crate::source_map::SourceMapGetter;
 use crate::Extension;
 use crate::ExtensionFileSource;
-use crate::ExtensionFileSourceCode;
 use crate::NoopModuleLoader;
 use crate::OpMiddlewareFn;
 use crate::OpResult;
@@ -869,14 +868,11 @@ impl JsRuntime {
       {
         let js_files = ext.get_js_sources();
         for file_source in js_files {
-          let ExtensionFileSourceCode::IncludedInBinary(code) =
-            file_source.code;
-
           // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
           realm.execute_script(
             self.v8_isolate(),
             &file_source.specifier,
-            code,
+            &file_source.code.load()?,
           )?;
         }
       }

core/snapshot_util.rs

@@ -33,6 +33,12 @@ pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) {
     snapshot_module_load_cb: create_snapshot_options.snapshot_module_load_cb,
     ..Default::default()
   });
+  println!(
+    "JsRuntime for snapshot prepared, took {:#?} ({})",
+    Instant::now().saturating_duration_since(mark),
+    create_snapshot_options.snapshot_path.display()
+  );
+  mark = Instant::now();
 
   let snapshot = js_runtime.snapshot();
   let snapshot_slice: &[u8] = &snapshot;

runtime/Cargo.toml

@@ -12,8 +12,17 @@ description = "Provides the deno runtime library"
 [features]
 # "fake" feature that allows to generate docs on docs.rs
 docsrs = []
-# feature that modifies the snapshot to allow extending it
+# A feature that disables creation of startup snapshot during in the build script.
+dont_create_runtime_snapshot = []
+# A feature that changes how startup snapshot is generated, that allows
+# extending it in embedder crates.
 snapshot_from_snapshot = []
+# A feature that disables embedding of the JavaScript source files in the binary.
+# With this feature enabled, the sources must be consumed during build time,
+# by creating a startup snapshot.
+include_js_files_for_snapshotting = [
+  "deno_core/include_js_files_for_snapshotting",
+]
 
 [lib]
 name = "deno_runtime"

runtime/build.rs

@@ -1,24 +1,25 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-use deno_core::include_js_files;
 use std::env;
 use std::path::PathBuf;
 
-// This is a shim that allows to generate documentation on docs.rs
-mod not_docs {
+#[cfg(all(
+  not(feature = "docsrs"),
+  not(feature = "dont_create_runtime_snapshot")
+))]
+mod startup_snapshot {
   use std::path::Path;
 
   use super::*;
-  use deno_cache::SqliteBackedCache;
-  use deno_core::snapshot_util::*;
-  use deno_core::Extension;
-
   use deno_ast::MediaType;
   use deno_ast::ParseParams;
   use deno_ast::SourceTextInfo;
+  use deno_cache::SqliteBackedCache;
   use deno_core::error::AnyError;
+  use deno_core::include_js_files;
+  use deno_core::snapshot_util::*;
+  use deno_core::Extension;
   use deno_core::ExtensionFileSource;
-  use deno_core::ExtensionFileSourceCode;
 
   fn transpile_ts_for_snapshotting(
     file_source: &ExtensionFileSource,
@@ -34,16 +35,15 @@ mod not_docs {
         file_source.specifier
       ),
     };
-
-    let ExtensionFileSourceCode::IncludedInBinary(code) = file_source.code;
+    let code = file_source.code.load()?;
 
     if !should_transpile {
-      return Ok(code.to_string());
+      return Ok(code);
     }
 
     let parsed = deno_ast::parse_module(ParseParams {
       specifier: file_source.specifier.to_string(),
-      text_info: SourceTextInfo::from_string(code.to_string()),
+      text_info: SourceTextInfo::from_string(code),
       media_type,
       capture_tokens: false,
       scope_analysis: false,
@@ -281,6 +281,7 @@ mod not_docs {
 
     #[cfg(not(feature = "snapshot_from_snapshot"))]
     {
+      use deno_core::ExtensionFileSourceCode;
       maybe_additional_extension = Some(
         Extension::builder("runtime_main")
           .dependencies(vec!["runtime"])
@@ -314,9 +315,13 @@ fn main() {
   // doesn't actually compile on docs.rs
   if env::var_os("DOCS_RS").is_some() {
     let snapshot_slice = &[];
+    #[allow(clippy::needless_borrow)]
     std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap();
   }
 
-  #[cfg(not(feature = "docsrs"))]
-  not_docs::build_snapshot(runtime_snapshot_path)
+  #[cfg(all(
+    not(feature = "docsrs"),
+    not(feature = "dont_create_runtime_snapshot")
+  ))]
+  startup_snapshot::build_snapshot(runtime_snapshot_path)
 }

runtime/js.rs

@@ -1,16 +1,21 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 use deno_core::Snapshot;
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 use log::debug;
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 use once_cell::sync::Lazy;
 
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
+static COMPRESSED_RUNTIME_SNAPSHOT: &[u8] =
+  include_bytes!(concat!(env!("OUT_DIR"), "/RUNTIME_SNAPSHOT.bin"));
+
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 pub static RUNTIME_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
   #[allow(clippy::uninit_vec)]
   #[cold]
   #[inline(never)]
   || {
-    static COMPRESSED_RUNTIME_SNAPSHOT: &[u8] =
-      include_bytes!(concat!(env!("OUT_DIR"), "/RUNTIME_SNAPSHOT.bin"));
-
     let size =
       u32::from_le_bytes(COMPRESSED_RUNTIME_SNAPSHOT[0..4].try_into().unwrap())
         as usize;
@@ -29,10 +34,15 @@ pub static RUNTIME_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
   },
 );
 
+#[cfg(not(feature = "dont_create_runtime_snapshot"))]
 pub fn deno_isolate_init() -> Snapshot {
   debug!("Deno isolate init with snapshots.");
   Snapshot::Static(&RUNTIME_SNAPSHOT)
 }
 
-#[cfg(feature = "snapshot_from_snapshot")]
+#[cfg(not(feature = "include_js_files_for_snapshotting"))]
 pub static SOURCE_CODE_FOR_99_MAIN_JS: &str = include_str!("js/99_main.js");
+
+#[cfg(feature = "include_js_files_for_snapshotting")]
+pub static PATH_FOR_99_MAIN_JS: &str =
+  concat!(env!("CARGO_MANIFEST_DIR"), "/js/99_main.js");

runtime/web_worker.rs

@@ -1,7 +1,6 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 use crate::colors;
 use crate::inspector_server::InspectorServer;
-use crate::js;
 use crate::ops;
 use crate::ops::io::Stdio;
 use crate::permissions::PermissionsContainer;
@@ -453,13 +452,17 @@ impl WebWorker {
     // Append exts
     extensions.extend(std::mem::take(&mut options.extensions));
 
+    #[cfg(not(feature = "dont_create_runtime_snapshot"))]
+    let startup_snapshot = options
+      .startup_snapshot
+      .unwrap_or_else(crate::js::deno_isolate_init);
+    #[cfg(feature = "dont_create_runtime_snapshot")]
+    let startup_snapshot = options.startup_snapshot
+      .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature.");
+
     let mut js_runtime = JsRuntime::new(RuntimeOptions {
       module_loader: Some(options.module_loader.clone()),
-      startup_snapshot: Some(
-        options
-          .startup_snapshot
-          .unwrap_or_else(js::deno_isolate_init),
-      ),
+      startup_snapshot: Some(startup_snapshot),
       source_map_getter: options.source_map_getter,
       get_error_class_fn: options.get_error_class_fn,
       shared_array_buffer_store: options.shared_array_buffer_store.clone(),

runtime/worker.rs

@@ -35,7 +35,6 @@ use deno_web::BlobStore;
 use log::debug;
 
 use crate::inspector_server::InspectorServer;
-use crate::js;
 use crate::ops;
 use crate::ops::io::Stdio;
 use crate::permissions::PermissionsContainer;
@@ -282,13 +281,17 @@ impl MainWorker {
     ];
     extensions.extend(std::mem::take(&mut options.extensions));
 
+    #[cfg(not(feature = "dont_create_runtime_snapshot"))]
+    let startup_snapshot = options
+      .startup_snapshot
+      .unwrap_or_else(crate::js::deno_isolate_init);
+    #[cfg(feature = "dont_create_runtime_snapshot")]
+    let startup_snapshot = options.startup_snapshot
+      .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature.");
+
     let mut js_runtime = JsRuntime::new(RuntimeOptions {
       module_loader: Some(options.module_loader.clone()),
-      startup_snapshot: Some(
-        options
-          .startup_snapshot
-          .unwrap_or_else(js::deno_isolate_init),
-      ),
+      startup_snapshot: Some(startup_snapshot),
       source_map_getter: options.source_map_getter,
       get_error_class_fn: options.get_error_class_fn,
       shared_array_buffer_store: options.shared_array_buffer_store.clone(),