1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-12-23 15:49:44 -05:00

feat(cli, runtime): compress snapshots (#13320)

This commit is contained in:
evan 2022-01-10 18:51:23 -05:00 committed by GitHub
parent a3b3a792b5
commit b66afa2518
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 161 additions and 9 deletions

52
Cargo.lock generated
View file

@ -369,6 +369,9 @@ name = "cc"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
dependencies = [
"jobserver",
]
[[package]]
name = "cfg-if"
@ -744,6 +747,7 @@ dependencies = [
"walkdir",
"winapi 0.3.9",
"winres",
"zstd",
]
[[package]]
@ -979,6 +983,7 @@ dependencies = [
"hyper",
"libc",
"log",
"lzzzz",
"netif",
"nix",
"notify",
@ -1985,6 +1990,15 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "jobserver"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.49"
@ -2212,6 +2226,15 @@ dependencies = [
"syn 1.0.65",
]
[[package]]
name = "lzzzz"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6d891cedd3b1659c206a60ff8afd15bccd7c2754b157f8a164861989e042b88"
dependencies = [
"cc",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
@ -5079,3 +5102,32 @@ dependencies = [
"syn 1.0.65",
"synstructure",
]
[[package]]
name = "zstd"
version = "0.9.2+zstd.1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "4.1.3+zstd.1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "1.6.2+zstd.1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
dependencies = [
"cc",
"libc",
]

View file

@ -88,6 +88,12 @@ opt-level = 3
opt-level = 3
[profile.bench.package.tokio]
opt-level = 3
[profile.bench.package.zstd]
opt-level = 3
[profile.bench.package.lzzzz]
opt-level = 3
[profile.bench.package.zstd-sys]
opt-level = 3
# NB: the `bench` and `release` profiles must remain EXACTLY the same.
[profile.release.package.rand]
@ -130,3 +136,9 @@ opt-level = 3
opt-level = 3
[profile.release.package.tokio]
opt-level = 3
[profile.release.package.zstd]
opt-level = 3
[profile.release.package.lzzzz]
opt-level = 3
[profile.release.package.zstd-sys]
opt-level = 3

View file

@ -33,6 +33,7 @@ deno_websocket = { version = "0.36.0", path = "../ext/websocket" }
deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" }
regex = "=1.5.4"
serde = { version = "=1.0.133", features = ["derive"] }
zstd = '=0.9.2'
[target.'cfg(windows)'.build-dependencies]
winapi = "=0.3.9"
@ -85,6 +86,7 @@ text_lines = "=0.4.1"
tokio = { version = "=1.14", features = ["full"] }
uuid = { version = "=0.8.2", features = ["v4", "serde"] }
walkdir = "=2.3.2"
zstd = '=0.9.2'
[target.'cfg(windows)'.dependencies]
fwdansi = "=1.1.0"

View file

@ -38,7 +38,30 @@ fn create_snapshot(
let snapshot = js_runtime.snapshot();
let snapshot_slice: &[u8] = &*snapshot;
println!("Snapshot size: {}", snapshot_slice.len());
std::fs::write(&snapshot_path, snapshot_slice).unwrap();
let compressed_snapshot_with_size = {
let mut vec = vec![];
vec.extend_from_slice(
&u32::try_from(snapshot.len())
.expect("snapshot larger than 4gb")
.to_le_bytes(),
);
vec.extend_from_slice(
&zstd::block::compress(snapshot_slice, 22)
.expect("snapshot compression failed"),
);
vec
};
println!(
"Snapshot compressed size: {}",
compressed_snapshot_with_size.len()
);
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
println!("Snapshot written to: {} ", snapshot_path.display());
}

View file

@ -51,11 +51,25 @@ pub static SHARED_GLOBALS_LIB: &str =
pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
pub static COMPILER_SNAPSHOT: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
#[cold]
#[inline(never)]
|| {
static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
zstd::block::decompress(
&COMPRESSED_COMPILER_SNAPSHOT[4..],
u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
as usize,
)
.unwrap()
.into_boxed_slice()
},
);
pub fn compiler_snapshot() -> Snapshot {
Snapshot::Static(COMPILER_SNAPSHOT)
Snapshot::Static(&*COMPILER_SNAPSHOT)
}
macro_rules! inc {

View file

@ -39,6 +39,8 @@ deno_webidl = { version = "0.31.0", path = "../ext/webidl" }
deno_websocket = { version = "0.36.0", path = "../ext/websocket" }
deno_webstorage = { version = "0.26.0", path = "../ext/webstorage" }
lzzzz = '=0.8.0'
[target.'cfg(windows)'.build-dependencies]
winres = "0.1.11"
winapi = "0.3.9"
@ -70,6 +72,7 @@ http = "0.2.4"
hyper = { version = "0.14.12", features = ["server", "stream", "http1", "http2", "runtime"] }
libc = "0.2.106"
log = "0.4.14"
lzzzz = '=0.8.0'
netif = "0.1.0"
notify = "=5.0.0-pre.12"
once_cell = "=1.9.0"

View file

@ -37,7 +37,32 @@ mod not_docs {
let snapshot = js_runtime.snapshot();
let snapshot_slice: &[u8] = &*snapshot;
println!("Snapshot size: {}", snapshot_slice.len());
std::fs::write(&snapshot_path, snapshot_slice).unwrap();
let compressed_snapshot_with_size = {
let mut vec = vec![];
vec.extend_from_slice(
&u32::try_from(snapshot.len())
.expect("snapshot larger than 4gb")
.to_le_bytes(),
);
lzzzz::lz4_hc::compress_to_vec(
snapshot_slice,
&mut vec,
lzzzz::lz4_hc::CLEVEL_MAX,
)
.expect("snapshot compression failed");
vec
};
println!(
"Snapshot compressed size: {}",
compressed_snapshot_with_size.len()
);
std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
println!("Snapshot written to: {} ", snapshot_path.display());
}

View file

@ -1,14 +1,35 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use deno_core::Snapshot;
use log::debug;
use once_cell::sync::Lazy;
pub static CLI_SNAPSHOT: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
pub static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
#[cold]
#[inline(never)]
|| {
static COMPRESSED_CLI_SNAPSHOT: &[u8] =
include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
let size =
u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
as usize;
let mut vec = Vec::with_capacity(size);
// SAFETY: vec is allocated with exact snapshot size (+ alignment)
// SAFETY: non zeroed bytes are overwritten with decompressed snapshot
unsafe {
vec.set_len(size);
}
lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
vec.into_boxed_slice()
},
);
pub fn deno_isolate_init() -> Snapshot {
debug!("Deno isolate init with snapshots.");
let data = CLI_SNAPSHOT;
Snapshot::Static(data)
Snapshot::Static(&*CLI_SNAPSHOT)
}
#[cfg(test)]