From bd034e360d036d9634d8526a6eea73979b3ad9e1 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Mon, 28 Aug 2023 22:30:46 +0100 Subject: [PATCH] refactor(runtime): factor out code between build.rs and worker.rs (#20299) Adds `runtime/shared.rs` which is imported by both `build.rs` and the rest of the crate, containing utilities used by both. Renames the `snapshot_from_snapshot` feature to `exclude_runtime_main_js` since that's what it does and it's relevant outside of snapshotting when `__runtime_js_sources` is specified. --- cli/Cargo.toml | 4 +- runtime/Cargo.toml | 5 +- runtime/build.rs | 120 ++---------------------------------------- runtime/lib.rs | 4 +- runtime/shared.rs | 107 +++++++++++++++++++++++++++++++++++++ runtime/web_worker.rs | 4 +- runtime/worker.rs | 76 +------------------------- 7 files changed, 123 insertions(+), 197 deletions(-) create mode 100644 runtime/shared.rs diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 72d75293e3..69635141bf 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -32,7 +32,7 @@ path = "./bench/lsp_bench_standalone.rs" __runtime_js_sources = ["deno_runtime/__runtime_js_sources"] [build-dependencies] -deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] } +deno_runtime = { workspace = true, features = ["exclude_runtime_main_js", "include_js_files_for_snapshotting"] } deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } lazy-regex.workspace = true serde.workspace = true @@ -55,7 +55,7 @@ deno_graph = "=0.52.0" deno_lint = { version = "=0.50.2", features = ["docs"] } deno_lockfile.workspace = true deno_npm.workspace = true -deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] } +deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "exclude_runtime_main_js", "include_js_files_for_snapshotting"] } deno_semver.workspace = true deno_task_shell = "=0.13.2" eszip = "=0.50.0" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index f181ba1c27..d4401effd3 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -14,9 +14,8 @@ description = "Provides the deno runtime library" docsrs = [] # A feature that disables creation of startup snapshot during in the build script. dont_create_runtime_snapshot = [] -# A feature that changes how startup snapshot is generated, that allows -# extending it in embedder crates. -snapshot_from_snapshot = [] +# A feature that allows excluding `./js/99_main.js` from the exported extension. +exclude_runtime_main_js = [] # A feature that disables embedding of the JavaScript source files in the binary. # With this feature enabled, the sources must be consumed during build time, # by creating a startup snapshot. diff --git a/runtime/build.rs b/runtime/build.rs index 56f9d611ac..5134c9d7a3 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,5 +1,7 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +mod shared; + use std::env; use std::path::PathBuf; @@ -9,59 +11,15 @@ use std::path::PathBuf; ))] mod startup_snapshot { use super::*; - use deno_ast::MediaType; - use deno_ast::ParseParams; - use deno_ast::SourceTextInfo; use deno_cache::SqliteBackedCache; use deno_core::error::AnyError; use deno_core::snapshot_util::*; use deno_core::Extension; - use deno_core::ExtensionFileSource; - use deno_core::ExtensionFileSourceCode; use deno_http::DefaultHttpPropertyExtractor; + use shared::maybe_transpile_source; + use shared::runtime; use std::path::Path; - // Duplicated in `worker.rs`. Keep in sync! - fn maybe_transpile_source( - source: &mut ExtensionFileSource, - ) -> Result<(), AnyError> { - // Always transpile `node:` built-in modules, since they might be TypeScript. - let media_type = if source.specifier.starts_with("node:") { - MediaType::TypeScript - } else { - MediaType::from_path(Path::new(&source.specifier)) - }; - - match media_type { - MediaType::TypeScript => {} - MediaType::JavaScript => return Ok(()), - MediaType::Mjs => return Ok(()), - _ => panic!( - "Unsupported media type for snapshotting {media_type:?} for file {}", - source.specifier - ), - } - let code = source.load()?; - - let parsed = deno_ast::parse_module(ParseParams { - specifier: source.specifier.to_string(), - text_info: SourceTextInfo::from_string(code.as_str().to_owned()), - media_type, - capture_tokens: false, - scope_analysis: false, - maybe_syntax: None, - })?; - let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { - imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, - inline_source_map: false, - ..Default::default() - })?; - - source.code = - ExtensionFileSourceCode::Computed(transpiled_source.text.into()); - Ok(()) - } - #[derive(Clone)] struct Permissions; @@ -241,71 +199,6 @@ mod startup_snapshot { } } - // Duplicated in `worker.rs`. Keep in sync! - deno_core::extension!(runtime, - deps = [ - deno_webidl, - deno_console, - deno_url, - deno_tls, - deno_web, - deno_fetch, - deno_cache, - deno_websocket, - deno_webstorage, - deno_crypto, - deno_broadcast_channel, - // FIXME(bartlomieju): this should be reenabled - // "deno_node", - deno_ffi, - deno_net, - deno_napi, - deno_http, - deno_io, - deno_fs - ], - esm = [ - dir "js", - "01_errors.js", - "01_version.ts", - "06_util.js", - "10_permissions.js", - "11_workers.js", - "13_buffer.js", - "30_os.js", - "40_fs_events.js", - "40_http.js", - "40_process.js", - "40_signals.js", - "40_tty.js", - "41_prompt.js", - "90_deno_ns.js", - "98_global_scope.js" - ], - ); - - #[cfg(not(feature = "snapshot_from_snapshot"))] - deno_core::extension!( - runtime_main, - deps = [runtime], - esm_entry_point = "ext:runtime_main/js/99_main.js", - customizer = |ext: &mut deno_core::Extension| { - ext.esm_files.to_mut().push(ExtensionFileSource { - specifier: "ext:runtime_main/js/99_main.js", - code: deno_core::ExtensionFileSourceCode::IncludedInBinary( - include_str!("js/99_main.js"), - ), - }); - } - ); - - #[cfg(feature = "snapshot_from_snapshot")] - deno_core::extension!( - runtime_main, - deps = [runtime], - esm_entry_point = "ext:runtime/90_deno_ns.js", - ); - pub fn create_runtime_snapshot(snapshot_path: PathBuf) { // NOTE(bartlomieju): ordering is important here, keep it in sync with // `runtime/worker.rs`, `runtime/web_worker.rs` and `cli/build.rs`! @@ -347,11 +240,8 @@ mod startup_snapshot { deno_http::deno_http::init_ops_and_esm::(), deno_io::deno_io::init_ops_and_esm(Default::default()), deno_fs::deno_fs::init_ops_and_esm::(false, fs.clone()), - runtime::init_ops_and_esm(), - // FIXME(bartlomieju): these extensions are specified last, because they - // depend on `runtime`, even though it should be other way around deno_node::deno_node::init_ops_and_esm::(None, fs), - runtime_main::init_ops_and_esm(), + runtime::init_ops_and_esm(), ]; for extension in &mut extensions { diff --git a/runtime/lib.rs b/runtime/lib.rs index 248ef94dea..bc49a2fefa 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -34,6 +34,8 @@ pub mod web_worker; pub mod worker; mod worker_bootstrap; -pub use worker::runtime; pub use worker_bootstrap::BootstrapOptions; pub use worker_bootstrap::WorkerLogLevel; + +mod shared; +pub use shared::runtime; diff --git a/runtime/shared.rs b/runtime/shared.rs new file mode 100644 index 0000000000..e7ba109b09 --- /dev/null +++ b/runtime/shared.rs @@ -0,0 +1,107 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// Utilities shared between `build.rs` and the rest of the crate. + +use deno_ast::MediaType; +use deno_ast::ParseParams; +use deno_ast::SourceTextInfo; +use deno_core::error::AnyError; +use deno_core::extension; +use deno_core::Extension; +use deno_core::ExtensionFileSource; +use deno_core::ExtensionFileSourceCode; +use std::path::Path; + +extension!(runtime, + deps = [ + deno_webidl, + deno_console, + deno_url, + deno_tls, + deno_web, + deno_fetch, + deno_cache, + deno_websocket, + deno_webstorage, + deno_crypto, + deno_broadcast_channel, + deno_node, + deno_ffi, + deno_net, + deno_napi, + deno_http, + deno_io, + deno_fs + ], + esm_entry_point = "ext:runtime/90_deno_ns.js", + esm = [ + dir "js", + "01_errors.js", + "01_version.ts", + "06_util.js", + "10_permissions.js", + "11_workers.js", + "13_buffer.js", + "30_os.js", + "40_fs_events.js", + "40_http.js", + "40_process.js", + "40_signals.js", + "40_tty.js", + "41_prompt.js", + "90_deno_ns.js", + "98_global_scope.js" + ], + customizer = |ext: &mut Extension| { + #[cfg(not(feature = "exclude_runtime_main_js"))] + { + ext.esm_files.to_mut().push(ExtensionFileSource { + specifier: "ext:runtime_main/js/99_main.js", + code: ExtensionFileSourceCode::IncludedInBinary( + include_str!("./js/99_main.js"), + ), + }); + ext.esm_entry_point = Some("ext:runtime_main/js/99_main.js"); + } + } +); + +#[allow(dead_code)] +pub fn maybe_transpile_source( + source: &mut ExtensionFileSource, +) -> Result<(), AnyError> { + // Always transpile `node:` built-in modules, since they might be TypeScript. + let media_type = if source.specifier.starts_with("node:") { + MediaType::TypeScript + } else { + MediaType::from_path(Path::new(&source.specifier)) + }; + + match media_type { + MediaType::TypeScript => {} + MediaType::JavaScript => return Ok(()), + MediaType::Mjs => return Ok(()), + _ => panic!( + "Unsupported media type for snapshotting {media_type:?} for file {}", + source.specifier + ), + } + let code = source.load()?; + + let parsed = deno_ast::parse_module(ParseParams { + specifier: source.specifier.to_string(), + text_info: SourceTextInfo::from_string(code.as_str().to_owned()), + media_type, + capture_tokens: false, + scope_analysis: false, + maybe_syntax: None, + })?; + let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { + imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, + inline_source_map: false, + ..Default::default() + })?; + + source.code = + ExtensionFileSourceCode::Computed(transpiled_source.text.into()); + Ok(()) +} diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 8a88dfa409..17bbb41eaa 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -3,8 +3,8 @@ use crate::colors; use crate::inspector_server::InspectorServer; use crate::ops; use crate::permissions::PermissionsContainer; +use crate::shared::runtime; use crate::tokio_util::create_and_run_current_thread; -use crate::worker::runtime; use crate::worker::FormatJsErrorFn; use crate::BootstrapOptions; use deno_broadcast_channel::InMemoryBroadcastChannel; @@ -484,7 +484,7 @@ impl WebWorker { } #[cfg(feature = "__runtime_js_sources")] { - use crate::worker::maybe_transpile_source; + use crate::shared::maybe_transpile_source; for source in extension.esm_files.to_mut() { maybe_transpile_source(source).unwrap(); } diff --git a/runtime/worker.rs b/runtime/worker.rs index a31bd2ae1b..6b9fca5614 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -18,8 +18,6 @@ use deno_core::futures::Future; use deno_core::v8; use deno_core::CompiledWasmModuleStore; use deno_core::Extension; -#[cfg(feature = "__runtime_js_sources")] -use deno_core::ExtensionFileSource; use deno_core::FsModuleLoader; use deno_core::GetErrorClassFn; use deno_core::JsRuntime; @@ -44,6 +42,7 @@ use log::debug; use crate::inspector_server::InspectorServer; use crate::ops; use crate::permissions::PermissionsContainer; +use crate::shared::runtime; use crate::BootstrapOptions; pub type FormatJsErrorFn = dyn Fn(&JsError) -> String + Sync + Send; @@ -61,78 +60,6 @@ impl ExitCode { } } -// Duplicated in `build.rs`. Keep in sync! -deno_core::extension!( - runtime, - esm_entry_point = "ext:runtime/90_deno_ns.js", - esm = [ - dir "js", - "01_errors.js", - "01_version.ts", - "06_util.js", - "10_permissions.js", - "11_workers.js", - "13_buffer.js", - "30_os.js", - "40_fs_events.js", - "40_http.js", - "40_process.js", - "40_signals.js", - "40_tty.js", - "41_prompt.js", - "90_deno_ns.js", - "98_global_scope.js" - ], -); - -// Duplicated in `build.rs`. Keep in sync! -#[cfg(feature = "__runtime_js_sources")] -pub fn maybe_transpile_source( - source: &mut ExtensionFileSource, -) -> Result<(), AnyError> { - use deno_ast::MediaType; - use deno_ast::ParseParams; - use deno_ast::SourceTextInfo; - use deno_core::ExtensionFileSourceCode; - use std::path::Path; - - // Always transpile `node:` built-in modules, since they might be TypeScript. - let media_type = if source.specifier.starts_with("node:") { - MediaType::TypeScript - } else { - MediaType::from_path(Path::new(&source.specifier)) - }; - - match media_type { - MediaType::TypeScript => {} - MediaType::JavaScript => return Ok(()), - MediaType::Mjs => return Ok(()), - _ => panic!( - "Unsupported media type for snapshotting {media_type:?} for file {}", - source.specifier - ), - } - let code = source.load()?; - - let parsed = deno_ast::parse_module(ParseParams { - specifier: source.specifier.to_string(), - text_info: SourceTextInfo::from_string(code.as_str().to_owned()), - media_type, - capture_tokens: false, - scope_analysis: false, - maybe_syntax: None, - })?; - let transpiled_source = parsed.transpile(&deno_ast::EmitOptions { - imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove, - inline_source_map: false, - ..Default::default() - })?; - - source.code = - ExtensionFileSourceCode::Computed(transpiled_source.text.into()); - Ok(()) -} - /// This worker is created and used by almost all /// subcommands in Deno executable. /// @@ -380,6 +307,7 @@ impl MainWorker { } #[cfg(feature = "__runtime_js_sources")] { + use crate::shared::maybe_transpile_source; for source in extension.esm_files.to_mut() { maybe_transpile_source(source).unwrap(); }