diff --git a/Cargo.lock b/Cargo.lock index 54ec13ac08..5165c654e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -738,23 +738,14 @@ dependencies = [ "data-url", "deno_ast", "deno_bench_util", - "deno_broadcast_channel", - "deno_console", "deno_core", - "deno_crypto", "deno_doc", "deno_emit", - "deno_fetch", "deno_graph", "deno_lint", - "deno_net", "deno_runtime", + "deno_snapshots", "deno_task_shell", - "deno_url", - "deno_web", - "deno_webgpu", - "deno_websocket", - "deno_webstorage", "dissimilar", "dprint-plugin-json", "dprint-plugin-markdown", @@ -1116,6 +1107,19 @@ dependencies = [ "winres", ] +[[package]] +name = "deno_snapshots" +version = "0.0.0" +dependencies = [ + "deno_core", + "deno_runtime", + "lzzzz", + "once_cell", + "regex", + "serde", + "zstd", +] + [[package]] name = "deno_task_shell" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 95063652e4..12a8b2cce7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "ops", "runtime", "serde_v8", + "snapshots", "test_ffi", "test_util", "ext/broadcast_channel", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1ed342c6dc..eba22c3e97 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,17 +25,7 @@ harness = false path = "./bench/lsp_bench_standalone.rs" [build-dependencies] -deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" } -deno_console = { version = "0.58.0", path = "../ext/console" } -deno_core = { version = "0.140.0", path = "../core" } -deno_crypto = { version = "0.72.0", path = "../ext/crypto" } -deno_fetch = { version = "0.81.0", path = "../ext/fetch" } -deno_net = { version = "0.50.0", path = "../ext/net" } -deno_url = { version = "0.58.0", path = "../ext/url" } -deno_web = { version = "0.89.0", path = "../ext/web" } -deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" } -deno_websocket = { version = "0.63.0", path = "../ext/websocket" } -deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" } +deno_runtime = { version = "0.66.0", path = "../runtime" } regex = "=1.5.6" serde = { version = "=1.0.136", features = ["derive"] } zstd = '=0.11.1' @@ -52,6 +42,7 @@ deno_emit = "0.3.0" deno_graph = "0.28.0" deno_lint = { version = "0.31.0", features = ["docs"] } deno_runtime = { version = "0.66.0", path = "../runtime" } +deno_snapshots = { version = "0.0.0", path = "../snapshots" } deno_task_shell = "0.4.0" atty = "=0.2.14" diff --git a/cli/build.rs b/cli/build.rs index 1a4eaa4254..095efa22f7 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,323 +1,19 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use deno_core::error::custom_error; -use deno_core::error::AnyError; -use deno_core::op; -use deno_core::serde::Deserialize; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; -use deno_core::Extension; -use deno_core::JsRuntime; -use deno_core::OpState; -use deno_core::RuntimeOptions; -use regex::Regex; -use std::collections::HashMap; +use deno_runtime::deno_broadcast_channel; +use deno_runtime::deno_console; +use deno_runtime::deno_crypto; +use deno_runtime::deno_fetch; +use deno_runtime::deno_net; +use deno_runtime::deno_url; +use deno_runtime::deno_web; +use deno_runtime::deno_websocket; +use deno_runtime::deno_webstorage; + use std::env; use std::path::Path; use std::path::PathBuf; -// TODO(bartlomieju): this module contains a lot of duplicated -// logic with `runtime/build.rs`, factor out to `deno_core`. -fn create_snapshot( - mut js_runtime: JsRuntime, - snapshot_path: &Path, - files: Vec, -) { - // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the - // workspace root. - let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); - for file in files { - println!("cargo:rerun-if-changed={}", file.display()); - let display_path = file.strip_prefix(display_root).unwrap(); - let display_path_str = display_path.display().to_string(); - js_runtime - .execute_script( - &("deno:".to_string() + &display_path_str.replace('\\', "/")), - &std::fs::read_to_string(&file).unwrap(), - ) - .unwrap(); - } - - let snapshot = js_runtime.snapshot(); - let snapshot_slice: &[u8] = &*snapshot; - println!("Snapshot size: {}", snapshot_slice.len()); - - let compressed_snapshot_with_size = { - let mut vec = vec![]; - - vec.extend_from_slice( - &u32::try_from(snapshot.len()) - .expect("snapshot larger than 4gb") - .to_le_bytes(), - ); - - vec.extend_from_slice( - &zstd::bulk::compress(snapshot_slice, 22) - .expect("snapshot compression failed"), - ); - - vec - }; - - println!( - "Snapshot compressed size: {}", - compressed_snapshot_with_size.len() - ); - - std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); - println!("Snapshot written to: {} ", snapshot_path.display()); -} - -#[derive(Debug, Deserialize)] -struct LoadArgs { - /// The fully qualified specifier that should be loaded. - specifier: String, -} - -fn create_compiler_snapshot( - snapshot_path: &Path, - files: Vec, - cwd: &Path, -) { - // libs that are being provided by op crates. - let mut op_crate_libs = HashMap::new(); - op_crate_libs.insert("deno.console", deno_console::get_declaration()); - op_crate_libs.insert("deno.url", deno_url::get_declaration()); - op_crate_libs.insert("deno.web", deno_web::get_declaration()); - op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); - op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration()); - op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration()); - op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration()); - op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration()); - op_crate_libs.insert( - "deno.broadcast_channel", - deno_broadcast_channel::get_declaration(), - ); - op_crate_libs.insert("deno.net", deno_net::get_declaration()); - - // ensure we invalidate the build properly. - for (_, path) in op_crate_libs.iter() { - println!("cargo:rerun-if-changed={}", path.display()); - } - - // libs that should be loaded into the isolate before snapshotting. - let libs = vec![ - // Deno custom type libraries - "deno.window", - "deno.worker", - "deno.shared_globals", - "deno.ns", - "deno.unstable", - // Deno built-in type libraries - "es5", - "es2015.collection", - "es2015.core", - "es2015", - "es2015.generator", - "es2015.iterable", - "es2015.promise", - "es2015.proxy", - "es2015.reflect", - "es2015.symbol", - "es2015.symbol.wellknown", - "es2016.array.include", - "es2016", - "es2017", - "es2017.intl", - "es2017.object", - "es2017.sharedmemory", - "es2017.string", - "es2017.typedarrays", - "es2018.asyncgenerator", - "es2018.asynciterable", - "es2018", - "es2018.intl", - "es2018.promise", - "es2018.regexp", - "es2019.array", - "es2019", - "es2019.object", - "es2019.string", - "es2019.symbol", - "es2020.bigint", - "es2020", - "es2020.date", - "es2020.intl", - "es2020.number", - "es2020.promise", - "es2020.sharedmemory", - "es2020.string", - "es2020.symbol.wellknown", - "es2021", - "es2021.intl", - "es2021.promise", - "es2021.string", - "es2021.weakref", - "es2022", - "es2022.array", - "es2022.error", - "es2022.intl", - "es2022.object", - "es2022.string", - "esnext", - "esnext.array", - "esnext.intl", - ]; - - let path_dts = cwd.join("dts"); - // ensure we invalidate the build properly. - for name in libs.iter() { - println!( - "cargo:rerun-if-changed={}", - path_dts.join(format!("lib.{}.d.ts", name)).display() - ); - } - - // create a copy of the vector that includes any op crate libs to be passed - // to the JavaScript compiler to build into the snapshot - let mut build_libs = libs.clone(); - for (op_lib, _) in op_crate_libs.iter() { - build_libs.push(op_lib.to_owned()); - } - - #[op] - fn op_build_info(state: &mut OpState) -> Value { - let build_specifier = "asset:///bootstrap.ts"; - let build_libs = state.borrow::>(); - json!({ - "buildSpecifier": build_specifier, - "libs": build_libs, - }) - } - - #[op] - fn op_cwd() -> String { - "cache:///".into() - } - - #[op] - fn op_exists() -> bool { - false - } - - #[op] - fn op_script_version( - _state: &mut OpState, - _args: Value, - ) -> Result, AnyError> { - Ok(Some("1".to_string())) - } - - #[op] - // using the same op that is used in `tsc.rs` for loading modules and reading - // files, but a slightly different implementation at build time. - fn op_load(state: &mut OpState, args: LoadArgs) -> Result { - let op_crate_libs = state.borrow::>(); - let path_dts = state.borrow::(); - let re_asset = - Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); - let build_specifier = "asset:///bootstrap.ts"; - - // we need a basic file to send to tsc to warm it up. - if args.specifier == build_specifier { - Ok(json!({ - "data": r#"console.log("hello deno!");"#, - "version": "1", - // this corresponds to `ts.ScriptKind.TypeScript` - "scriptKind": 3 - })) - // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to - // parse out just the name so we can lookup the asset. - } else if let Some(caps) = re_asset.captures(&args.specifier) { - if let Some(lib) = caps.get(1).map(|m| m.as_str()) { - // if it comes from an op crate, we were supplied with the path to the - // file. - let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { - PathBuf::from(op_crate_lib).canonicalize().unwrap() - // otherwise we are will generate the path ourself - } else { - path_dts.join(format!("lib.{}.d.ts", lib)) - }; - let data = std::fs::read_to_string(path)?; - Ok(json!({ - "data": data, - "version": "1", - // this corresponds to `ts.ScriptKind.TypeScript` - "scriptKind": 3 - })) - } else { - Err(custom_error( - "InvalidSpecifier", - format!("An invalid specifier was requested: {}", args.specifier), - )) - } - } else { - Err(custom_error( - "InvalidSpecifier", - format!("An invalid specifier was requested: {}", args.specifier), - )) - } - } - let js_runtime = JsRuntime::new(RuntimeOptions { - will_snapshot: true, - extensions: vec![Extension::builder() - .ops(vec![ - op_build_info::decl(), - op_cwd::decl(), - op_exists::decl(), - op_load::decl(), - op_script_version::decl(), - ]) - .state(move |state| { - state.put(op_crate_libs.clone()); - state.put(build_libs.clone()); - state.put(path_dts.clone()); - - Ok(()) - }) - .build()], - ..Default::default() - }); - - create_snapshot(js_runtime, snapshot_path, files); -} - -fn ts_version() -> String { - std::fs::read_to_string("tsc/00_typescript.js") - .unwrap() - .lines() - .find(|l| l.contains("ts.version = ")) - .expect( - "Failed to find the pattern `ts.version = ` in typescript source code", - ) - .chars() - .skip_while(|c| !char::is_numeric(*c)) - .take_while(|c| *c != '"') - .collect::() -} - -fn git_commit_hash() -> String { - if let Ok(output) = std::process::Command::new("git") - .arg("rev-list") - .arg("-1") - .arg("HEAD") - .output() - { - if output.status.success() { - std::str::from_utf8(&output.stdout[..40]) - .unwrap() - .to_string() - } else { - // When not in git repository - // (e.g. when the user install by `cargo install deno`) - "UNKNOWN".to_string() - } - } else { - // When there is no git command for some reason - "UNKNOWN".to_string() - } -} - fn main() { // Skip building from docs.rs. if env::var_os("DOCS_RS").is_some() { @@ -330,8 +26,9 @@ fn main() { if target != host { panic!("Cross compiling with snapshot is not supported."); } - // To debug snapshot issues uncomment: - // op_fetch_asset::trace_serializer(); + + println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); + println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); if let Ok(c) = env::var("DENO_CANARY") { println!("cargo:rustc-env=DENO_CANARY={}", c); @@ -385,18 +82,6 @@ fn main() { deno_net::get_declaration().display() ); - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); - println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - - // Main snapshot - let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin"); - - let js_files = get_js_files("tsc"); - create_compiler_snapshot(&compiler_snapshot_path, js_files, &c); - #[cfg(target_os = "windows")] { let mut res = winres::WindowsResource::new(); @@ -414,16 +99,38 @@ fn deno_webgpu_get_declaration() -> PathBuf { manifest_dir.join("dts").join("lib.deno_webgpu.d.ts") } -fn get_js_files(d: &str) -> Vec { - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let mut js_files = std::fs::read_dir(d) - .unwrap() - .map(|dir_entry| { - let file = dir_entry.unwrap(); - manifest_dir.join(file.path()) - }) - .filter(|path| path.extension().unwrap_or_default() == "js") - .collect::>(); - js_files.sort(); - js_files +fn git_commit_hash() -> String { + if let Ok(output) = std::process::Command::new("git") + .arg("rev-list") + .arg("-1") + .arg("HEAD") + .output() + { + if output.status.success() { + std::str::from_utf8(&output.stdout[..40]) + .unwrap() + .to_string() + } else { + // When not in git repository + // (e.g. when the user install by `cargo install deno`) + "UNKNOWN".to_string() + } + } else { + // When there is no git command for some reason + "UNKNOWN".to_string() + } +} + +fn ts_version() -> String { + std::fs::read_to_string("tsc/00_typescript.js") + .unwrap() + .lines() + .find(|l| l.contains("ts.version = ")) + .expect( + "Failed to find the pattern `ts.version = ` in typescript source code", + ) + .chars() + .skip_while(|c| !char::is_numeric(*c)) + .take_while(|c| *c != '"') + .collect::() } diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 6ff4cba00f..2a5bdc2d80 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -2618,7 +2618,7 @@ fn op_script_version( fn js_runtime(performance: Arc) -> JsRuntime { JsRuntime::new(RuntimeOptions { extensions: vec![init_extension(performance)], - startup_snapshot: Some(tsc::compiler_snapshot()), + startup_snapshot: Some(deno_snapshots::tsc_snapshot()), ..Default::default() }) } diff --git a/cli/main.rs b/cli/main.rs index 471afdffdb..01dbba848e 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -179,6 +179,7 @@ fn create_web_worker_callback( shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), stdio: stdio.clone(), + startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; WebWorker::bootstrap_from_options( @@ -274,6 +275,7 @@ pub fn create_main_worker( shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), stdio, + startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; MainWorker::bootstrap_from_options(main_module, permissions, options) diff --git a/cli/standalone.rs b/cli/standalone.rs index f19329cbd4..1e8429db04 100644 --- a/cli/standalone.rs +++ b/cli/standalone.rs @@ -307,6 +307,7 @@ pub async fn run( shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), + startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), diff --git a/cli/tsc.rs b/cli/tsc.rs index a61d7871df..b293ea3b2c 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -24,7 +24,6 @@ use deno_core::JsRuntime; use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::RuntimeOptions; -use deno_core::Snapshot; use deno_graph::Resolved; use once_cell::sync::Lazy; use std::collections::HashMap; @@ -52,27 +51,6 @@ pub static SHARED_GLOBALS_LIB: &str = pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts"); pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts"); -pub static COMPILER_SNAPSHOT: Lazy> = Lazy::new( - #[cold] - #[inline(never)] - || { - static COMPRESSED_COMPILER_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); - - zstd::bulk::decompress( - &COMPRESSED_COMPILER_SNAPSHOT[4..], - u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap()) - as usize, - ) - .unwrap() - .into_boxed_slice() - }, -); - -pub fn compiler_snapshot() -> Snapshot { - Snapshot::Static(&*COMPILER_SNAPSHOT) -} - macro_rules! inc { ($e:expr) => { include_str!(concat!("dts/", $e)) @@ -657,7 +635,7 @@ pub fn exec(request: Request) -> Result { }) .collect(); let mut runtime = JsRuntime::new(RuntimeOptions { - startup_snapshot: Some(compiler_snapshot()), + startup_snapshot: Some(deno_snapshots::tsc_snapshot()), extensions: vec![Extension::builder() .ops(vec![ op_cwd::decl(), @@ -841,9 +819,9 @@ mod tests { } #[test] - fn test_compiler_snapshot() { + fn test_tsc_snapshot() { let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { - startup_snapshot: Some(compiler_snapshot()), + startup_snapshot: Some(deno_snapshots::tsc_snapshot()), ..Default::default() }); js_runtime diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 7cb7b78200..652b2342d7 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,25 +21,6 @@ path = "lib.rs" name = "hello_runtime" path = "examples/hello_runtime.rs" -[build-dependencies] -deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" } -deno_console = { version = "0.58.0", path = "../ext/console" } -deno_core = { version = "0.140.0", path = "../core" } -deno_crypto = { version = "0.72.0", path = "../ext/crypto" } -deno_fetch = { version = "0.81.0", path = "../ext/fetch" } -deno_ffi = { version = "0.45.0", path = "../ext/ffi" } -deno_http = { version = "0.52.0", path = "../ext/http" } -deno_net = { version = "0.50.0", path = "../ext/net" } -deno_tls = { version = "0.45.0", path = "../ext/tls" } -deno_url = { version = "0.58.0", path = "../ext/url" } -deno_web = { version = "0.89.0", path = "../ext/web" } -deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" } -deno_webidl = { version = "0.58.0", path = "../ext/webidl" } -deno_websocket = { version = "0.63.0", path = "../ext/websocket" } -deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" } - -lzzzz = '1.0' - [target.'cfg(windows)'.build-dependencies] winres = "0.1.11" winapi = "0.3.9" diff --git a/runtime/build.rs b/runtime/build.rs index eea7a3602c..0cb113af1a 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,219 +1,17 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::env; -use std::path::Path; -use std::path::PathBuf; - -// This is a shim that allows to generate documentation on docs.rs -#[cfg(not(feature = "docsrs"))] -mod not_docs { - use super::*; - use deno_core::Extension; - use deno_core::JsRuntime; - use deno_core::RuntimeOptions; - - // TODO(bartlomieju): this module contains a lot of duplicated - // logic with `cli/build.rs`, factor out to `deno_core`. - fn create_snapshot( - mut js_runtime: JsRuntime, - snapshot_path: &Path, - files: Vec, - ) { - // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the - // workspace root. - let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); - for file in files { - println!("cargo:rerun-if-changed={}", file.display()); - let display_path = file.strip_prefix(display_root).unwrap(); - let display_path_str = display_path.display().to_string(); - js_runtime - .execute_script( - &("deno:".to_string() + &display_path_str.replace('\\', "/")), - &std::fs::read_to_string(&file).unwrap(), - ) - .unwrap(); - } - - let snapshot = js_runtime.snapshot(); - let snapshot_slice: &[u8] = &*snapshot; - println!("Snapshot size: {}", snapshot_slice.len()); - - let compressed_snapshot_with_size = { - let mut vec = vec![]; - - vec.extend_from_slice( - &u32::try_from(snapshot.len()) - .expect("snapshot larger than 4gb") - .to_le_bytes(), - ); - - lzzzz::lz4_hc::compress_to_vec( - snapshot_slice, - &mut vec, - lzzzz::lz4_hc::CLEVEL_MAX, - ) - .expect("snapshot compression failed"); - - vec - }; - - println!( - "Snapshot compressed size: {}", - compressed_snapshot_with_size.len() - ); - - std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); - println!("Snapshot written to: {} ", snapshot_path.display()); - } - - struct Permissions; - - impl deno_fetch::FetchPermissions for Permissions { - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_read( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - } - - impl deno_websocket::WebSocketPermissions for Permissions { - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - } - - impl deno_web::TimersPermission for Permissions { - fn allow_hrtime(&mut self) -> bool { - unreachable!("snapshotting!") - } - - fn check_unstable( - &self, - _state: &deno_core::OpState, - _api_name: &'static str, - ) { - unreachable!("snapshotting!") - } - } - - impl deno_ffi::FfiPermissions for Permissions { - fn check( - &mut self, - _path: Option<&Path>, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - } - - impl deno_net::NetPermissions for Permissions { - fn check_net>( - &mut self, - _host: &(T, Option), - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_read( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_write( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - } - - fn create_runtime_snapshot(snapshot_path: &Path, files: Vec) { - let extensions: Vec = vec![ - deno_webidl::init(), - deno_console::init(), - deno_url::init(), - deno_tls::init(), - deno_web::init::( - deno_web::BlobStore::default(), - Default::default(), - ), - deno_fetch::init::(Default::default()), - deno_websocket::init::("".to_owned(), None, None), - deno_webstorage::init(None), - deno_crypto::init(None), - deno_webgpu::init(false), - deno_broadcast_channel::init( - deno_broadcast_channel::InMemoryBroadcastChannel::default(), - false, // No --unstable. - ), - deno_ffi::init::(false), - deno_net::init::( - None, false, // No --unstable. - None, - ), - deno_http::init(), - ]; - - let js_runtime = JsRuntime::new(RuntimeOptions { - will_snapshot: true, - extensions, - ..Default::default() - }); - create_snapshot(js_runtime, snapshot_path, files); - } - - fn get_js_files(d: &str) -> Vec { - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let mut js_files = std::fs::read_dir(d) - .unwrap() - .map(|dir_entry| { - let file = dir_entry.unwrap(); - manifest_dir.join(file.path()) - }) - .filter(|path| path.extension().unwrap_or_default() == "js") - .collect::>(); - js_files.sort(); - js_files - } - - pub fn build_snapshot(runtime_snapshot_path: PathBuf) { - let js_files = get_js_files("js"); - create_runtime_snapshot(&runtime_snapshot_path, js_files); - } -} - fn main() { - // To debug snapshot issues uncomment: - // op_fetch_asset::trace_serializer(); - - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); - println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - - // Main snapshot - let runtime_snapshot_path = o.join("CLI_SNAPSHOT.bin"); - - // If we're building on docs.rs we just create - // and empty snapshot file and return, because `rusty_v8` - // doesn't actually compile on docs.rs - if env::var_os("DOCS_RS").is_some() { - let snapshot_slice = &[]; - std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap(); + // Skip building from docs.rs. + if std::env::var_os("DOCS_RS").is_some() { return; } - #[cfg(not(feature = "docsrs"))] - not_docs::build_snapshot(runtime_snapshot_path) + println!( + "cargo:rustc-env=TARGET={}", + std::env::var("TARGET").unwrap() + ); + println!( + "cargo:rustc-env=PROFILE={}", + std::env::var("PROFILE").unwrap() + ); } diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs index 07e42f0ffa..19f462d4f7 100644 --- a/runtime/examples/hello_runtime.rs +++ b/runtime/examples/hello_runtime.rs @@ -58,6 +58,7 @@ async fn main() -> Result<(), AnyError> { shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), + startup_snapshot: None, }; let js_path = diff --git a/runtime/js.rs b/runtime/js.rs index cdd479858e..a18dc3bd3b 100644 --- a/runtime/js.rs +++ b/runtime/js.rs @@ -1,58 +1,38 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use deno_core::Snapshot; -use log::debug; -use once_cell::sync::Lazy; +use deno_core::include_js_files; +use deno_core::Extension; -pub static CLI_SNAPSHOT: Lazy> = Lazy::new( - #[allow(clippy::uninit_vec)] - #[cold] - #[inline(never)] - || { - static COMPRESSED_CLI_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); - - let size = - u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap()) - as usize; - let mut vec = Vec::with_capacity(size); - - // SAFETY: vec is allocated with exact snapshot size (+ alignment) - // SAFETY: non zeroed bytes are overwritten with decompressed snapshot - unsafe { - vec.set_len(size); - } - - lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap(); - - vec.into_boxed_slice() - }, -); - -pub fn deno_isolate_init() -> Snapshot { - debug!("Deno isolate init with snapshots."); - Snapshot::Static(&*CLI_SNAPSHOT) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn cli_snapshot() { - let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { - startup_snapshot: Some(deno_isolate_init()), - ..Default::default() - }); - js_runtime - .execute_script( - "", - r#" - if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) { - throw Error("bad"); - } - console.log("we have console.log!!!"); - "#, - ) - .unwrap(); - } +pub fn init() -> Extension { + Extension::builder() + .js(include_js_files!( + prefix "deno:runtime", + // Generated with: + // bash -c "cd runtime && ls js/*.js | sort" + "js/01_build.js", + "js/01_errors.js", + "js/01_version.js", + "js/01_web_util.js", + "js/06_util.js", + "js/10_permissions.js", + "js/11_workers.js", + "js/12_io.js", + "js/13_buffer.js", + "js/30_fs.js", + "js/30_os.js", + "js/40_diagnostics.js", + "js/40_files.js", + "js/40_fs_events.js", + "js/40_http.js", + "js/40_process.js", + "js/40_read_file.js", + "js/40_signals.js", + "js/40_spawn.js", + "js/40_testing.js", + "js/40_tty.js", + "js/40_write_file.js", + "js/41_prompt.js", + "js/90_deno_ns.js", + "js/99_main.js", + )) + .build() } diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index ba2c016cc9..19e344ee7e 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -335,6 +335,7 @@ pub struct WebWorkerOptions { pub shared_array_buffer_store: Option, pub compiled_wasm_module_store: Option, pub stdio: Stdio, + pub startup_snapshot: Option, } impl WebWorker { @@ -427,6 +428,8 @@ impl WebWorker { ops::tty::init(), deno_http::init(), ops::http::init(), + // Runtime JS + js::init(), // Permissions ext (worker specific state) perm_ext, ]; @@ -436,7 +439,7 @@ impl WebWorker { let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: Some(js::deno_isolate_init()), + startup_snapshot: options.startup_snapshot.take(), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), diff --git a/runtime/worker.rs b/runtime/worker.rs index e46b5c75ff..acb50dc305 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -81,6 +81,7 @@ pub struct WorkerOptions { pub shared_array_buffer_store: Option, pub compiled_wasm_module_store: Option, pub stdio: Stdio, + pub startup_snapshot: Option, } impl MainWorker { @@ -168,6 +169,8 @@ impl MainWorker { ops::tty::init(), deno_http::init(), ops::http::init(), + // Runtime JS + js::init(), // Permissions ext (worker specific state) perm_ext, ]; @@ -175,7 +178,7 @@ impl MainWorker { let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: Some(js::deno_isolate_init()), + startup_snapshot: options.startup_snapshot.take(), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), @@ -411,6 +414,7 @@ mod tests { shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), + startup_snapshot: None, }; MainWorker::bootstrap_from_options(main_module, permissions, options) diff --git a/snapshots/Cargo.toml b/snapshots/Cargo.toml new file mode 100644 index 0000000000..6266ed681c --- /dev/null +++ b/snapshots/Cargo.toml @@ -0,0 +1,32 @@ +# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. + +# IMPORTANT(bartlomieju): this crate is internal and shouldn't be published +# to crates.io + +[package] +name = "deno_snapshots" +version = "0.0.0" +authors = ["the Deno authors"] +edition = "2018" +license = "MIT" +repository = "https://github.com/denoland/deno" +description = "Provides snapshots of TSC & Deno (runtime+web+core)" + +[lib] +name = "deno_snapshots" +path = "lib.rs" + +[dependencies] +deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s +deno_runtime = { version = "0.66.0", path = "../runtime" } +lzzzz = "1.0" +once_cell = "1.10.0" +zstd = "0.11.1" + +[build-dependencies] +deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s +deno_runtime = { version = "0.66.0", path = "../runtime" } +lzzzz = "1.0" +regex = "1.5.6" +serde = { version = "1.0.125", features = ["derive"] } +zstd = "0.11.1" diff --git a/snapshots/build.rs b/snapshots/build.rs new file mode 100644 index 0000000000..df868591d6 --- /dev/null +++ b/snapshots/build.rs @@ -0,0 +1,10 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +mod build_runtime; +mod build_tsc; + +fn main() { + let out_dir = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); + build_runtime::create_runtime_snapshot(&out_dir.join("CLI_SNAPSHOT.bin")); + build_tsc::create_tsc_snapshot(&out_dir.join("COMPILER_SNAPSHOT.bin")); +} diff --git a/snapshots/build_runtime.rs b/snapshots/build_runtime.rs new file mode 100644 index 0000000000..ae1c673226 --- /dev/null +++ b/snapshots/build_runtime.rs @@ -0,0 +1,168 @@ +// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. +use std::convert::TryFrom; +use std::path::Path; + +use deno_runtime::deno_broadcast_channel; +use deno_runtime::deno_console; +use deno_runtime::deno_core; +use deno_runtime::deno_crypto; +use deno_runtime::deno_fetch; +use deno_runtime::deno_ffi; +use deno_runtime::deno_http; +use deno_runtime::deno_net; +use deno_runtime::deno_tls; +use deno_runtime::deno_url; +use deno_runtime::deno_web; +use deno_runtime::deno_webgpu; +use deno_runtime::deno_webidl; +use deno_runtime::deno_websocket; +use deno_runtime::deno_webstorage; + +use deno_core::Extension; +use deno_core::JsRuntime; +use deno_core::RuntimeOptions; + +pub fn create_runtime_snapshot(snapshot_path: &Path) { + let extensions: Vec = vec![ + deno_webidl::init(), + deno_console::init(), + deno_url::init(), + deno_tls::init(), + deno_web::init::( + deno_web::BlobStore::default(), + Default::default(), + ), + deno_fetch::init::(Default::default()), + deno_websocket::init::("".to_owned(), None, None), + deno_webstorage::init(None), + deno_crypto::init(None), + deno_webgpu::init(false), + deno_broadcast_channel::init( + deno_broadcast_channel::InMemoryBroadcastChannel::default(), + false, // No --unstable. + ), + deno_ffi::init::(false), + deno_net::init::( + None, false, // No --unstable. + None, + ), + deno_http::init(), + // Runtime JS + deno_runtime::js::init(), + ]; + + let js_runtime = JsRuntime::new(RuntimeOptions { + will_snapshot: true, + extensions, + ..Default::default() + }); + write_runtime_snapshot(js_runtime, snapshot_path); +} + +// TODO(bartlomieju): this module contains a lot of duplicated +// logic with `build_tsc.rs` +fn write_runtime_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) { + let snapshot = js_runtime.snapshot(); + let snapshot_slice: &[u8] = &*snapshot; + println!("Snapshot size: {}", snapshot_slice.len()); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + lzzzz::lz4_hc::compress_to_vec( + snapshot_slice, + &mut vec, + lzzzz::lz4_hc::CLEVEL_MAX, + ) + .expect("snapshot compression failed"); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); + println!("Snapshot written to: {} ", snapshot_path.display()); +} + +struct Permissions; + +impl deno_fetch::FetchPermissions for Permissions { + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_read( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } +} + +impl deno_websocket::WebSocketPermissions for Permissions { + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } +} + +impl deno_web::TimersPermission for Permissions { + fn allow_hrtime(&mut self) -> bool { + unreachable!("snapshotting!") + } + + fn check_unstable( + &self, + _state: &deno_core::OpState, + _api_name: &'static str, + ) { + unreachable!("snapshotting!") + } +} + +impl deno_ffi::FfiPermissions for Permissions { + fn check( + &mut self, + _path: Option<&Path>, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } +} + +impl deno_net::NetPermissions for Permissions { + fn check_net>( + &mut self, + _host: &(T, Option), + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_read( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_write( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } +} diff --git a/snapshots/build_tsc.rs b/snapshots/build_tsc.rs new file mode 100644 index 0000000000..5296ae23bf --- /dev/null +++ b/snapshots/build_tsc.rs @@ -0,0 +1,328 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use deno_runtime::deno_broadcast_channel; +use deno_runtime::deno_console; +use deno_runtime::deno_crypto; +use deno_runtime::deno_fetch; +use deno_runtime::deno_net; +use deno_runtime::deno_url; +use deno_runtime::deno_web; +use deno_runtime::deno_websocket; +use deno_runtime::deno_webstorage; + +use deno_runtime::deno_core::error::custom_error; +use deno_runtime::deno_core::error::AnyError; +use deno_runtime::deno_core::op; +use deno_runtime::deno_core::serde::Deserialize; +use deno_runtime::deno_core::serde_json::json; +use deno_runtime::deno_core::serde_json::Value; +use deno_runtime::deno_core::Extension; +use deno_runtime::deno_core::JsRuntime; +use deno_runtime::deno_core::OpState; +use deno_runtime::deno_core::RuntimeOptions; + +use regex::Regex; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::env; +use std::path::Path; +use std::path::PathBuf; + +pub fn create_tsc_snapshot(snapshot_path: &Path) { + let mut js_runtime = JsRuntime::new(RuntimeOptions { + will_snapshot: true, + extensions: vec![tsc_snapshot_init()], + ..Default::default() + }); + load_js_files(&mut js_runtime); + write_snapshot(js_runtime, snapshot_path); +} + +// TODO(bartlomieju): this module contains a lot of duplicated +// logic with `build_runtime.rs` +fn write_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) { + let snapshot = js_runtime.snapshot(); + let snapshot_slice: &[u8] = &*snapshot; + println!("Snapshot size: {}", snapshot_slice.len()); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + vec.extend_from_slice( + &zstd::bulk::compress(snapshot_slice, 22) + .expect("snapshot compression failed"), + ); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); + println!("Snapshot written to: {} ", snapshot_path.display()); +} + +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + +fn tsc_snapshot_init() -> Extension { + // libs that are being provided by op crates. + let mut op_crate_libs = HashMap::new(); + op_crate_libs.insert("deno.console", deno_console::get_declaration()); + op_crate_libs.insert("deno.url", deno_url::get_declaration()); + op_crate_libs.insert("deno.web", deno_web::get_declaration()); + op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); + op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration()); + op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration()); + op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration()); + op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration()); + op_crate_libs.insert( + "deno.broadcast_channel", + deno_broadcast_channel::get_declaration(), + ); + op_crate_libs.insert("deno.net", deno_net::get_declaration()); + + // ensure we invalidate the build properly. + for (_, path) in op_crate_libs.iter() { + println!("cargo:rerun-if-changed={}", path.display()); + } + + // libs that should be loaded into the isolate before snapshotting. + let libs = vec![ + // Deno custom type libraries + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.ns", + "deno.unstable", + // Deno built-in type libraries + "es5", + "es2015.collection", + "es2015.core", + "es2015", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2016", + "es2017", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020", + "es2020.date", + "es2020.intl", + "es2020.number", + "es2020.promise", + "es2020.sharedmemory", + "es2020.string", + "es2020.symbol.wellknown", + "es2021", + "es2021.intl", + "es2021.promise", + "es2021.string", + "es2021.weakref", + "es2022", + "es2022.array", + "es2022.error", + "es2022.intl", + "es2022.object", + "es2022.string", + "esnext", + "esnext.array", + "esnext.intl", + ]; + + let cli_dir = cli_dir(); + let path_dts = cli_dir.join("dts"); + // ensure we invalidate the build properly. + for name in libs.iter() { + println!( + "cargo:rerun-if-changed={}", + path_dts.join(format!("lib.{}.d.ts", name)).display() + ); + } + + // create a copy of the vector that includes any op crate libs to be passed + // to the JavaScript compiler to build into the snapshot + let mut build_libs = libs.clone(); + for (op_lib, _) in op_crate_libs.iter() { + build_libs.push(op_lib.to_owned()); + } + + #[op] + fn op_build_info(state: &mut OpState) -> Value { + let build_specifier = "asset:///bootstrap.ts"; + let build_libs = state.borrow::>(); + json!({ + "buildSpecifier": build_specifier, + "libs": build_libs, + }) + } + + #[op] + fn op_cwd() -> String { + "cache:///".into() + } + + #[op] + fn op_exists() -> bool { + false + } + + #[op] + fn op_script_version( + _state: &mut OpState, + _args: Value, + ) -> Result, AnyError> { + Ok(Some("1".to_string())) + } + + #[op] + // using the same op that is used in `tsc.rs` for loading modules and reading + // files, but a slightly different implementation at build time. + fn op_load(state: &mut OpState, args: LoadArgs) -> Result { + let op_crate_libs = state.borrow::>(); + let path_dts = state.borrow::(); + let re_asset = + Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); + let build_specifier = "asset:///bootstrap.ts"; + + // we need a basic file to send to tsc to warm it up. + if args.specifier == build_specifier { + Ok(json!({ + "data": r#"console.log("hello deno!");"#, + "version": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to + // parse out just the name so we can lookup the asset. + } else if let Some(caps) = re_asset.captures(&args.specifier) { + if let Some(lib) = caps.get(1).map(|m| m.as_str()) { + // if it comes from an op crate, we were supplied with the path to the + // file. + let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { + PathBuf::from(op_crate_lib).canonicalize().unwrap() + // otherwise we are will generate the path ourself + } else { + path_dts.join(format!("lib.{}.d.ts", lib)) + }; + let data = std::fs::read_to_string(path)?; + Ok(json!({ + "data": data, + "version": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", args.specifier), + )) + } + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", args.specifier), + )) + } + } + + Extension::builder() + .ops(vec![ + op_build_info::decl(), + op_cwd::decl(), + op_exists::decl(), + op_load::decl(), + op_script_version::decl(), + ]) + .state(move |state| { + state.put(op_crate_libs.clone()); + state.put(build_libs.clone()); + state.put(path_dts.clone()); + + Ok(()) + }) + .build() +} + +fn deno_webgpu_get_declaration() -> PathBuf { + cli_dir().join("dts").join("lib.deno_webgpu.d.ts") +} + +fn load_js_files(js_runtime: &mut JsRuntime) { + let js_files = get_js_files(tsc_dir()); + let cwd = cli_dir(); + let display_root = cwd.parent().unwrap(); + for file in js_files { + println!("cargo:rerun-if-changed={}", file.display()); + let display_path = file.strip_prefix(display_root).unwrap(); + let display_path_str = display_path.display().to_string(); + js_runtime + .execute_script( + &("deno:".to_string() + &display_path_str.replace('\\', "/")), + &std::fs::read_to_string(&file).unwrap(), + ) + .unwrap(); + } +} + +fn root_dir() -> PathBuf { + // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the workspace root. + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .canonicalize() + .unwrap() +} + +fn cli_dir() -> PathBuf { + root_dir().join("cli") +} + +fn tsc_dir() -> PathBuf { + cli_dir().join("tsc") +} + +fn get_js_files(dir: PathBuf) -> Vec { + let mut js_files = std::fs::read_dir(dir.clone()) + .unwrap() + .map(|dir_entry| { + let file = dir_entry.unwrap(); + dir.join(file.path()) + }) + .filter(|path| path.extension().unwrap_or_default() == "js") + .collect::>(); + js_files.sort(); + js_files +} diff --git a/snapshots/lib.rs b/snapshots/lib.rs new file mode 100644 index 0000000000..1d0b3ceb3f --- /dev/null +++ b/snapshots/lib.rs @@ -0,0 +1,77 @@ +use deno_core::Snapshot; +use once_cell::sync::Lazy; +use std::convert::TryInto; + +pub fn tsc_snapshot() -> Snapshot { + Snapshot::Static(&*COMPILER_SNAPSHOT) +} + +static COMPILER_SNAPSHOT: Lazy> = Lazy::new( + #[cold] + #[inline(never)] + || { + static COMPRESSED_COMPILER_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); + + zstd::bulk::decompress( + &COMPRESSED_COMPILER_SNAPSHOT[4..], + u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap()) + as usize, + ) + .unwrap() + .into_boxed_slice() + }, +); + +pub fn cli_snapshot() -> Snapshot { + Snapshot::Static(&*CLI_SNAPSHOT) +} + +static CLI_SNAPSHOT: Lazy> = Lazy::new( + #[allow(clippy::uninit_vec)] + #[cold] + #[inline(never)] + || { + static COMPRESSED_CLI_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); + + let size = + u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap()) + as usize; + let mut vec = Vec::with_capacity(size); + + // SAFETY: vec is allocated with exact snapshot size (+ alignment) + // SAFETY: non zeroed bytes are overwritten with decompressed snapshot + unsafe { + vec.set_len(size); + } + + lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap(); + + vec.into_boxed_slice() + }, +); + +#[cfg(test)] +mod tests { + use deno_runtime::deno_core; + + #[test] + fn cli_snapshot() { + let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { + startup_snapshot: Some(crate::cli_snapshot()), + ..Default::default() + }); + js_runtime + .execute_script( + "", + r#" + if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) { + throw Error("bad"); + } + console.log("we have console.log!!!"); + "#, + ) + .unwrap(); + } +}