feat(cli, runtime): compress snapshots (#13320)

Cargo.lock

@@ -369,6 +369,9 @@ name = "cc"
 version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
+dependencies = [
+ "jobserver",
+]
 
 [[package]]
 name = "cfg-if"
@@ -744,6 +747,7 @@ dependencies = [
  "walkdir",
  "winapi 0.3.9",
  "winres",
+ "zstd",
 ]
 
 [[package]]
@@ -979,6 +983,7 @@ dependencies = [
  "hyper",
  "libc",
  "log",
+ "lzzzz",
  "netif",
  "nix",
  "notify",
@@ -1985,6 +1990,15 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
+[[package]]
+name = "jobserver"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.49"
@@ -2212,6 +2226,15 @@ dependencies = [
  "syn 1.0.65",
 ]
 
+[[package]]
+name = "lzzzz"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6d891cedd3b1659c206a60ff8afd15bccd7c2754b157f8a164861989e042b88"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"
@@ -5079,3 +5102,32 @@ dependencies = [
  "syn 1.0.65",
  "synstructure",
 ]
+
+[[package]]
+name = "zstd"
+version = "0.9.2+zstd.1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54"
+dependencies = [
+ "zstd-safe",
+]
+
+[[package]]
+name = "zstd-safe"
+version = "4.1.3+zstd.1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79"
+dependencies = [
+ "libc",
+ "zstd-sys",
+]
+
+[[package]]
+name = "zstd-sys"
+version = "1.6.2+zstd.1.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
+dependencies = [
+ "cc",
+ "libc",
+]

Cargo.toml

@@ -88,6 +88,12 @@ opt-level = 3
 opt-level = 3
 [profile.bench.package.tokio]
 opt-level = 3
+[profile.bench.package.zstd]
+opt-level = 3
+[profile.bench.package.lzzzz]
+opt-level = 3
+[profile.bench.package.zstd-sys]
+opt-level = 3
 
 # NB: the `bench` and `release` profiles must remain EXACTLY the same.
 [profile.release.package.rand]
@@ -130,3 +136,9 @@ opt-level = 3
 opt-level = 3
 [profile.release.package.tokio]
 opt-level = 3
+[profile.release.package.zstd]
+opt-level = 3
+[profile.release.package.lzzzz]
+opt-level = 3
+[profile.release.package.zstd-sys]
+opt-level = 3

cli/Cargo.toml

@@ -33,6 +33,7 @@ deno_websocket = { version = "0.36.0", path = "../ext/websocket" }
 deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" }
 regex = "=1.5.4"
 serde = { version = "=1.0.133", features = ["derive"] }
+zstd = '=0.9.2'
 
 [target.'cfg(windows)'.build-dependencies]
 winapi = "=0.3.9"
@@ -85,6 +86,7 @@ text_lines = "=0.4.1"
 tokio = { version = "=1.14", features = ["full"] }
 uuid = { version = "=0.8.2", features = ["v4", "serde"] }
 walkdir = "=2.3.2"
+zstd = '=0.9.2'
 
 [target.'cfg(windows)'.dependencies]
 fwdansi = "=1.1.0"

cli/build.rs

@@ -38,7 +38,30 @@ fn create_snapshot(
   let snapshot = js_runtime.snapshot();
   let snapshot_slice: &[u8] = &*snapshot;
   println!("Snapshot size: {}", snapshot_slice.len());
-  std::fs::write(&snapshot_path, snapshot_slice).unwrap();
+
+  let compressed_snapshot_with_size = {
+    let mut vec = vec![];
+
+    vec.extend_from_slice(
+      &u32::try_from(snapshot.len())
+        .expect("snapshot larger than 4gb")
+        .to_le_bytes(),
+    );
+
+    vec.extend_from_slice(
+      &zstd::block::compress(snapshot_slice, 22)
+        .expect("snapshot compression failed"),
+    );
+
+    vec
+  };
+
+  println!(
+    "Snapshot compressed size: {}",
+    compressed_snapshot_with_size.len()
+  );
+
+  std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
   println!("Snapshot written to: {} ", snapshot_path.display());
 }
 

cli/tsc.rs

@@ -51,11 +51,25 @@ pub static SHARED_GLOBALS_LIB: &str =
 pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
 pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
 
-pub static COMPILER_SNAPSHOT: &[u8] =
+pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+  #[cold]
+  #[inline(never)]
+  || {
+    static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
       include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
 
+    zstd::block::decompress(
+      &COMPRESSED_COMPILER_SNAPSHOT[4..],
+      u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
+        as usize,
+    )
+    .unwrap()
+    .into_boxed_slice()
+  },
+);
+
 pub fn compiler_snapshot() -> Snapshot {
-  Snapshot::Static(COMPILER_SNAPSHOT)
+  Snapshot::Static(&*COMPILER_SNAPSHOT)
 }
 
 macro_rules! inc {

runtime/Cargo.toml

@@ -39,6 +39,8 @@ deno_webidl = { version = "0.31.0", path = "../ext/webidl" }
 deno_websocket = { version = "0.36.0", path = "../ext/websocket" }
 deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" }
 
+lzzzz = '=0.8.0'
+
 [target.'cfg(windows)'.build-dependencies]
 winres = "0.1.11"
 winapi = "0.3.9"
@@ -70,6 +72,7 @@ http = "0.2.4"
 hyper = { version = "0.14.12", features = ["server", "stream", "http1", "http2", "runtime"] }
 libc = "0.2.106"
 log = "0.4.14"
+lzzzz = '=0.8.0'
 netif = "0.1.0"
 notify = "=5.0.0-pre.12"
 once_cell = "=1.9.0"

runtime/build.rs

@@ -37,7 +37,32 @@ mod not_docs {
     let snapshot = js_runtime.snapshot();
     let snapshot_slice: &[u8] = &*snapshot;
     println!("Snapshot size: {}", snapshot_slice.len());
-    std::fs::write(&snapshot_path, snapshot_slice).unwrap();
+
+    let compressed_snapshot_with_size = {
+      let mut vec = vec![];
+
+      vec.extend_from_slice(
+        &u32::try_from(snapshot.len())
+          .expect("snapshot larger than 4gb")
+          .to_le_bytes(),
+      );
+
+      lzzzz::lz4_hc::compress_to_vec(
+        snapshot_slice,
+        &mut vec,
+        lzzzz::lz4_hc::CLEVEL_MAX,
+      )
+      .expect("snapshot compression failed");
+
+      vec
+    };
+
+    println!(
+      "Snapshot compressed size: {}",
+      compressed_snapshot_with_size.len()
+    );
+
+    std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
     println!("Snapshot written to: {} ", snapshot_path.display());
   }
 

runtime/js.rs

@@ -1,14 +1,35 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 use deno_core::Snapshot;
 use log::debug;
+use once_cell::sync::Lazy;
 
-pub static CLI_SNAPSHOT: &[u8] =
+pub static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+  #[cold]
+  #[inline(never)]
+  || {
+    static COMPRESSED_CLI_SNAPSHOT: &[u8] =
       include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
 
+    let size =
+      u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
+        as usize;
+    let mut vec = Vec::with_capacity(size);
+
+    // SAFETY: vec is allocated with exact snapshot size (+ alignment)
+    // SAFETY: non zeroed bytes are overwritten with decompressed snapshot
+    unsafe {
+      vec.set_len(size);
+    }
+
+    lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
+
+    vec.into_boxed_slice()
+  },
+);
+
 pub fn deno_isolate_init() -> Snapshot {
   debug!("Deno isolate init with snapshots.");
-  let data = CLI_SNAPSHOT;
-  Snapshot::Static(data)
+  Snapshot::Static(&*CLI_SNAPSHOT)
 }
 
 #[cfg(test)]