From 678a881f63ee8e3b86d4dc539234a78c94cc9c1e Mon Sep 17 00:00:00 2001 From: Aaron O'Mullan Date: Tue, 5 Oct 2021 22:41:14 +0200 Subject: [PATCH] refactor(runtime): Worker bootstrap options (#12299) --- cli/main.rs | 61 +++++++++++------- cli/standalone.rs | 32 ++++++---- runtime/examples/hello_runtime.rs | 31 +++++---- runtime/lib.rs | 3 + runtime/web_worker.rs | 75 +++++++++------------- runtime/worker.rs | 100 ++++++++++++------------------ runtime/worker_bootstrap.rs | 48 ++++++++++++++ 7 files changed, 193 insertions(+), 157 deletions(-) create mode 100644 runtime/worker_bootstrap.rs diff --git a/cli/main.rs b/cli/main.rs index 52e3f6e955..758681ecbd 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -82,6 +82,7 @@ use deno_runtime::web_worker::WebWorker; use deno_runtime::web_worker::WebWorkerOptions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::BootstrapOptions; use log::debug; use log::info; use std::env; @@ -111,11 +112,21 @@ fn create_web_worker_callback(ps: ProcState) -> Arc { let create_web_worker_cb = create_web_worker_callback(ps.clone()); let options = WebWorkerOptions { - args: ps.flags.argv.clone(), - apply_source_maps: true, - debug_flag: ps.flags.log_level.map_or(false, |l| l == log::Level::Debug), - unstable: ps.flags.unstable, - enable_testing_features: ps.flags.enable_testing_features, + bootstrap: BootstrapOptions { + args: ps.flags.argv.clone(), + apply_source_maps: true, + cpu_count: num_cpus::get(), + debug_flag: ps + .flags + .log_level + .map_or(false, |l| l == log::Level::Debug), + enable_testing_features: ps.flags.enable_testing_features, + location: Some(args.main_module.clone()), + no_color: !colors::use_color(), + runtime_version: version::deno(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: ps.flags.unstable, + }, unsafely_ignore_certificate_errors: ps .flags .unsafely_ignore_certificate_errors @@ -129,25 +140,26 @@ fn create_web_worker_callback(ps: ProcState) -> Arc { use_deno_namespace: args.use_deno_namespace, worker_type: args.worker_type, maybe_inspector_server, - runtime_version: version::deno(), - ts_version: version::TYPESCRIPT.to_string(), - no_color: !colors::use_color(), get_error_class_fn: Some(&crate::errors::get_error_class_name), blob_store: ps.blob_store.clone(), broadcast_channel: ps.broadcast_channel.clone(), shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), - cpu_count: num_cpus::get(), }; + let bootstrap_options = options.bootstrap.clone(); + // TODO(@AaronO): switch to bootstrap_from_options() once ops below are an extension + // since it uses sync_ops_cache() which currently depends on the Deno namespace + // which can be nuked when bootstrapping workers (use_deno_namespace: false) let (mut worker, external_handle) = WebWorker::from_options( args.name, args.permissions, args.main_module, args.worker_id, - &options, + options, ); + // TODO(@AaronO): move to a JsRuntime Extension passed into options // This block registers additional ops and state that // are only available in the CLI { @@ -164,7 +176,7 @@ fn create_web_worker_callback(ps: ProcState) -> Arc { } js_runtime.sync_ops_cache(); } - worker.bootstrap(&options); + worker.bootstrap(&bootstrap_options); (worker, external_handle) }) @@ -192,11 +204,18 @@ pub fn create_main_worker( let create_web_worker_cb = create_web_worker_callback(ps.clone()); let options = WorkerOptions { - apply_source_maps: true, - args: ps.flags.argv.clone(), - debug_flag: ps.flags.log_level.map_or(false, |l| l == log::Level::Debug), - unstable: ps.flags.unstable, - enable_testing_features: ps.flags.enable_testing_features, + bootstrap: BootstrapOptions { + apply_source_maps: true, + args: ps.flags.argv.clone(), + cpu_count: num_cpus::get(), + debug_flag: ps.flags.log_level.map_or(false, |l| l == log::Level::Debug), + enable_testing_features: ps.flags.enable_testing_features, + location: ps.flags.location.clone(), + no_color: !colors::use_color(), + runtime_version: version::deno(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: ps.flags.unstable, + }, unsafely_ignore_certificate_errors: ps .flags .unsafely_ignore_certificate_errors @@ -209,11 +228,7 @@ pub fn create_main_worker( maybe_inspector_server, should_break_on_first_statement, module_loader, - runtime_version: version::deno(), - ts_version: version::TYPESCRIPT.to_string(), - no_color: !colors::use_color(), get_error_class_fn: Some(&crate::errors::get_error_class_name), - location: ps.flags.location.clone(), origin_storage_dir: ps.flags.location.clone().map(|loc| { ps.dir .root @@ -226,11 +241,12 @@ pub fn create_main_worker( broadcast_channel: ps.broadcast_channel.clone(), shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), - cpu_count: num_cpus::get(), }; - let mut worker = MainWorker::from_options(main_module, permissions, &options); + let mut worker = + MainWorker::bootstrap_from_options(main_module, permissions, options); + // TODO(@AaronO): move to a JsRuntime Extension passed into options // This block registers additional ops and state that // are only available in the CLI { @@ -250,7 +266,6 @@ pub fn create_main_worker( js_runtime.sync_ops_cache(); } - worker.bootstrap(&options); worker } diff --git a/cli/standalone.rs b/cli/standalone.rs index 914eeaddf1..2e4704ea9c 100644 --- a/cli/standalone.rs +++ b/cli/standalone.rs @@ -26,6 +26,7 @@ use deno_runtime::permissions::Permissions; use deno_runtime::permissions::PermissionsOptions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::BootstrapOptions; use deno_tls::create_default_root_cert_store; use log::Level; use std::convert::TryInto; @@ -228,12 +229,19 @@ pub async fn run( } let options = WorkerOptions { - apply_source_maps: false, - args: metadata.argv, - debug_flag: metadata.log_level.map_or(false, |l| l == log::Level::Debug), + bootstrap: BootstrapOptions { + apply_source_maps: false, + args: metadata.argv, + cpu_count: num_cpus::get(), + debug_flag: metadata.log_level.map_or(false, |l| l == log::Level::Debug), + enable_testing_features: false, + location: metadata.location, + no_color: !colors::use_color(), + runtime_version: version::deno(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: metadata.unstable, + }, user_agent: version::get_user_agent(), - unstable: metadata.unstable, - enable_testing_features: false, unsafely_ignore_certificate_errors: metadata .unsafely_ignore_certificate_errors, root_cert_store: Some(root_cert_store), @@ -243,20 +251,19 @@ pub async fn run( maybe_inspector_server: None, should_break_on_first_statement: false, module_loader, - runtime_version: version::deno(), - ts_version: version::TYPESCRIPT.to_string(), - no_color: !colors::use_color(), get_error_class_fn: Some(&get_error_class_name), - location: metadata.location, origin_storage_dir: None, blob_store, broadcast_channel, shared_array_buffer_store: None, compiled_wasm_module_store: None, - cpu_count: num_cpus::get(), }; - let mut worker = - MainWorker::from_options(main_module.clone(), permissions, &options); + let mut worker = MainWorker::bootstrap_from_options( + main_module.clone(), + permissions, + options, + ); + // TODO(@AaronO): move to a JsRuntime Extension passed into options { let js_runtime = &mut worker.js_runtime; js_runtime @@ -267,7 +274,6 @@ pub async fn run( ops::runtime_compiler::init(js_runtime); js_runtime.sync_ops_cache(); } - worker.bootstrap(&options); worker.execute_main_module(&main_module).await?; worker.execute_script( &located_script_name!(), diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs index eb6b5a5917..eb4557c049 100644 --- a/runtime/examples/hello_runtime.rs +++ b/runtime/examples/hello_runtime.rs @@ -7,6 +7,7 @@ use deno_runtime::deno_web::BlobStore; use deno_runtime::permissions::Permissions; use deno_runtime::worker::MainWorker; use deno_runtime::worker::WorkerOptions; +use deno_runtime::BootstrapOptions; use std::path::Path; use std::rc::Rc; use std::sync::Arc; @@ -23,11 +24,18 @@ async fn main() -> Result<(), AnyError> { }); let options = WorkerOptions { - apply_source_maps: false, - args: vec![], - debug_flag: false, - unstable: false, - enable_testing_features: false, + bootstrap: BootstrapOptions { + apply_source_maps: false, + args: vec![], + cpu_count: 1, + debug_flag: false, + enable_testing_features: false, + location: None, + no_color: false, + runtime_version: "x".to_string(), + ts_version: "x".to_string(), + unstable: false, + }, unsafely_ignore_certificate_errors: None, root_cert_store: None, user_agent: "hello_runtime".to_string(), @@ -37,17 +45,12 @@ async fn main() -> Result<(), AnyError> { maybe_inspector_server: None, should_break_on_first_statement: false, module_loader, - runtime_version: "x".to_string(), - ts_version: "x".to_string(), - no_color: false, get_error_class_fn: Some(&get_error_class_name), - location: None, origin_storage_dir: None, blob_store: BlobStore::default(), broadcast_channel: InMemoryBroadcastChannel::default(), shared_array_buffer_store: None, compiled_wasm_module_store: None, - cpu_count: 1, }; let js_path = @@ -55,9 +58,11 @@ async fn main() -> Result<(), AnyError> { let main_module = deno_core::resolve_path(&js_path.to_string_lossy())?; let permissions = Permissions::allow_all(); - let mut worker = - MainWorker::from_options(main_module.clone(), permissions, &options); - worker.bootstrap(&options); + let mut worker = MainWorker::bootstrap_from_options( + main_module.clone(), + permissions, + options, + ); worker.execute_main_module(&main_module).await?; worker.run_event_loop(false).await?; Ok(()) diff --git a/runtime/lib.rs b/runtime/lib.rs index 37d48def15..e89e4a27bc 100644 --- a/runtime/lib.rs +++ b/runtime/lib.rs @@ -27,3 +27,6 @@ pub mod permissions; pub mod tokio_util; pub mod web_worker; pub mod worker; + +mod worker_bootstrap; +pub use worker_bootstrap::BootstrapOptions; diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 1ca5deb7a8..3db74dc578 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -6,6 +6,7 @@ use crate::metrics; use crate::ops; use crate::permissions::Permissions; use crate::tokio_util::create_basic_runtime; +use crate::BootstrapOptions; use deno_broadcast_channel::InMemoryBroadcastChannel; use deno_core::error::AnyError; use deno_core::error::JsError; @@ -15,7 +16,6 @@ use deno_core::futures::stream::StreamExt; use deno_core::located_script_name; use deno_core::serde::Deserialize; use deno_core::serde::Serialize; -use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::v8; use deno_core::CancelHandle; @@ -35,7 +35,6 @@ use deno_web::BlobStore; use deno_web::MessagePort; use log::debug; use std::cell::RefCell; -use std::env; use std::fmt; use std::rc::Rc; use std::sync::atomic::AtomicBool; @@ -260,11 +259,7 @@ pub struct WebWorker { } pub struct WebWorkerOptions { - /// Sets `Deno.args` in JS runtime. - pub args: Vec, - pub debug_flag: bool, - pub unstable: bool, - pub enable_testing_features: bool, + pub bootstrap: BootstrapOptions, pub unsafely_ignore_certificate_errors: Option>, pub root_cert_store: Option, pub user_agent: String, @@ -275,32 +270,38 @@ pub struct WebWorkerOptions { pub use_deno_namespace: bool, pub worker_type: WebWorkerType, pub maybe_inspector_server: Option>, - pub apply_source_maps: bool, - /// Sets `Deno.version.deno` in JS runtime. - pub runtime_version: String, - /// Sets `Deno.version.typescript` in JS runtime. - pub ts_version: String, - /// Sets `Deno.noColor` in JS runtime. - pub no_color: bool, pub get_error_class_fn: Option, pub blob_store: BlobStore, pub broadcast_channel: InMemoryBroadcastChannel, pub shared_array_buffer_store: Option, pub compiled_wasm_module_store: Option, - pub cpu_count: usize, } impl WebWorker { + pub fn bootstrap_from_options( + name: String, + permissions: Permissions, + main_module: ModuleSpecifier, + worker_id: WorkerId, + options: WebWorkerOptions, + ) -> (Self, SendableWebWorkerHandle) { + let bootstrap_options = options.bootstrap.clone(); + let (mut worker, handle) = + Self::from_options(name, permissions, main_module, worker_id, options); + worker.bootstrap(&bootstrap_options); + (worker, handle) + } + pub fn from_options( name: String, permissions: Permissions, main_module: ModuleSpecifier, worker_id: WorkerId, - options: &WebWorkerOptions, + options: WebWorkerOptions, ) -> (Self, SendableWebWorkerHandle) { // Permissions: many ops depend on this - let unstable = options.unstable; - let enable_testing_features = options.enable_testing_features; + let unstable = options.bootstrap.unstable; + let enable_testing_features = options.bootstrap.enable_testing_features; let perm_ext = Extension::builder() .state(move |state| { state.put::(permissions.clone()); @@ -329,15 +330,12 @@ impl WebWorker { options.root_cert_store.clone(), options.unsafely_ignore_certificate_errors.clone(), ), - deno_broadcast_channel::init( - options.broadcast_channel.clone(), - options.unstable, - ), + deno_broadcast_channel::init(options.broadcast_channel.clone(), unstable), deno_crypto::init(options.seed), - deno_webgpu::init(options.unstable), + deno_webgpu::init(unstable), deno_timers::init::(), // ffi - deno_ffi::init::(options.unstable), + deno_ffi::init::(unstable), // Metrics metrics::init(), // Permissions ext (worker specific state) @@ -360,7 +358,7 @@ impl WebWorker { deno_tls::init(), deno_net::init::( options.root_cert_store.clone(), - options.unstable, + unstable, options.unsafely_ignore_certificate_errors.clone(), ), ops::os::init(), @@ -419,32 +417,15 @@ impl WebWorker { ) } - pub fn bootstrap(&mut self, options: &WebWorkerOptions) { - let runtime_options = json!({ - "args": options.args, - "applySourceMaps": options.apply_source_maps, - "debugFlag": options.debug_flag, - "denoVersion": options.runtime_version, - "noColor": options.no_color, - "pid": std::process::id(), - "ppid": ops::runtime::ppid(), - "target": env!("TARGET"), - "tsVersion": options.ts_version, - "unstableFlag": options.unstable, - "enableTestingFeaturesFlag": options.enable_testing_features, - "v8Version": deno_core::v8_version(), - "location": self.main_module, - "cpuCount": options.cpu_count, - }); - - let runtime_options_str = - serde_json::to_string_pretty(&runtime_options).unwrap(); - + pub fn bootstrap(&mut self, options: &BootstrapOptions) { // Instead of using name for log we use `worker-${id}` because // WebWorkers can have empty string as name. let script = format!( "bootstrap.workerRuntime({}, \"{}\", {}, \"{}\")", - runtime_options_str, self.name, options.use_deno_namespace, self.id + options.as_json(), + self.name, + self.use_deno_namespace, + self.id ); self .execute_script(&located_script_name!(), &script) diff --git a/runtime/worker.rs b/runtime/worker.rs index 709763c7d0..8327c0dd94 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -5,13 +5,11 @@ use crate::js; use crate::metrics; use crate::ops; use crate::permissions::Permissions; +use crate::BootstrapOptions; use deno_broadcast_channel::InMemoryBroadcastChannel; use deno_core::error::AnyError; use deno_core::futures::Future; use deno_core::located_script_name; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::url::Url; use deno_core::CompiledWasmModuleStore; use deno_core::Extension; use deno_core::GetErrorClassFn; @@ -26,7 +24,6 @@ use deno_core::SharedArrayBufferStore; use deno_tls::rustls::RootCertStore; use deno_web::BlobStore; use log::debug; -use std::env; use std::pin::Pin; use std::rc::Rc; use std::sync::Arc; @@ -46,12 +43,7 @@ pub struct MainWorker { } pub struct WorkerOptions { - pub apply_source_maps: bool, - /// Sets `Deno.args` in JS runtime. - pub args: Vec, - pub debug_flag: bool, - pub unstable: bool, - pub enable_testing_features: bool, + pub bootstrap: BootstrapOptions, pub unsafely_ignore_certificate_errors: Option>, pub root_cert_store: Option, pub user_agent: String, @@ -62,31 +54,34 @@ pub struct WorkerOptions { pub js_error_create_fn: Option>, pub maybe_inspector_server: Option>, pub should_break_on_first_statement: bool, - /// Sets `Deno.version.deno` in JS runtime. - pub runtime_version: String, - /// Sets `Deno.version.typescript` in JS runtime. - pub ts_version: String, - /// Sets `Deno.noColor` in JS runtime. - pub no_color: bool, pub get_error_class_fn: Option, - pub location: Option, pub origin_storage_dir: Option, pub blob_store: BlobStore, pub broadcast_channel: InMemoryBroadcastChannel, pub shared_array_buffer_store: Option, pub compiled_wasm_module_store: Option, - pub cpu_count: usize, } impl MainWorker { + pub fn bootstrap_from_options( + main_module: ModuleSpecifier, + permissions: Permissions, + options: WorkerOptions, + ) -> Self { + let bootstrap_options = options.bootstrap.clone(); + let mut worker = Self::from_options(main_module, permissions, options); + worker.bootstrap(&bootstrap_options); + worker + } + pub fn from_options( main_module: ModuleSpecifier, permissions: Permissions, - options: &WorkerOptions, + options: WorkerOptions, ) -> Self { // Permissions: many ops depend on this - let unstable = options.unstable; - let enable_testing_features = options.enable_testing_features; + let unstable = options.bootstrap.unstable; + let enable_testing_features = options.bootstrap.enable_testing_features; let perm_ext = Extension::builder() .state(move |state| { state.put::(permissions.clone()); @@ -102,7 +97,10 @@ impl MainWorker { deno_webidl::init(), deno_console::init(), deno_url::init(), - deno_web::init(options.blob_store.clone(), options.location.clone()), + deno_web::init( + options.blob_store.clone(), + options.bootstrap.location.clone(), + ), deno_fetch::init::( options.user_agent.clone(), options.root_cert_store.clone(), @@ -118,14 +116,11 @@ impl MainWorker { ), deno_webstorage::init(options.origin_storage_dir.clone()), deno_crypto::init(options.seed), - deno_broadcast_channel::init( - options.broadcast_channel.clone(), - options.unstable, - ), - deno_webgpu::init(options.unstable), + deno_broadcast_channel::init(options.broadcast_channel.clone(), unstable), + deno_webgpu::init(unstable), deno_timers::init::(), // ffi - deno_ffi::init::(options.unstable), + deno_ffi::init::(unstable), // Metrics metrics::init(), // Runtime ops @@ -138,7 +133,7 @@ impl MainWorker { deno_tls::init(), deno_net::init::( options.root_cert_store.clone(), - options.unstable, + unstable, options.unsafely_ignore_certificate_errors.clone(), ), ops::os::init(), @@ -173,27 +168,8 @@ impl MainWorker { } } - pub fn bootstrap(&mut self, options: &WorkerOptions) { - let runtime_options = json!({ - "args": options.args, - "applySourceMaps": options.apply_source_maps, - "debugFlag": options.debug_flag, - "denoVersion": options.runtime_version, - "noColor": options.no_color, - "pid": std::process::id(), - "ppid": ops::runtime::ppid(), - "target": env!("TARGET"), - "tsVersion": options.ts_version, - "unstableFlag": options.unstable, - "v8Version": deno_core::v8_version(), - "location": options.location, - "cpuCount": options.cpu_count, - }); - - let script = format!( - "bootstrap.mainRuntime({})", - serde_json::to_string_pretty(&runtime_options).unwrap() - ); + pub fn bootstrap(&mut self, options: &BootstrapOptions) { + let script = format!("bootstrap.mainRuntime({})", options.as_json()); self .execute_script(&located_script_name!(), &script) .expect("Failed to execute bootstrap script"); @@ -325,12 +301,19 @@ mod tests { let permissions = Permissions::default(); let options = WorkerOptions { - apply_source_maps: false, + bootstrap: BootstrapOptions { + apply_source_maps: false, + args: vec![], + cpu_count: 1, + debug_flag: false, + enable_testing_features: false, + location: None, + no_color: true, + runtime_version: "x".to_string(), + ts_version: "x".to_string(), + unstable: false, + }, user_agent: "x".to_string(), - args: vec![], - debug_flag: false, - unstable: false, - enable_testing_features: false, unsafely_ignore_certificate_errors: None, root_cert_store: None, seed: None, @@ -339,20 +322,15 @@ mod tests { maybe_inspector_server: None, should_break_on_first_statement: false, module_loader: Rc::new(deno_core::FsModuleLoader), - runtime_version: "x".to_string(), - ts_version: "x".to_string(), - no_color: true, get_error_class_fn: None, - location: None, origin_storage_dir: None, blob_store: BlobStore::default(), broadcast_channel: InMemoryBroadcastChannel::default(), shared_array_buffer_store: None, compiled_wasm_module_store: None, - cpu_count: 1, }; - MainWorker::from_options(main_module, permissions, &options) + MainWorker::bootstrap_from_options(main_module, permissions, options) } #[tokio::test] diff --git a/runtime/worker_bootstrap.rs b/runtime/worker_bootstrap.rs new file mode 100644 index 0000000000..9bd519d735 --- /dev/null +++ b/runtime/worker_bootstrap.rs @@ -0,0 +1,48 @@ +use crate::ops::runtime::ppid; +use deno_core::serde_json; +use deno_core::serde_json::json; +use deno_core::ModuleSpecifier; + +/// Common bootstrap options for MainWorker & WebWorker +#[derive(Clone)] +pub struct BootstrapOptions { + /// Sets `Deno.args` in JS runtime. + pub args: Vec, + pub apply_source_maps: bool, + pub cpu_count: usize, + pub debug_flag: bool, + pub enable_testing_features: bool, + pub location: Option, + /// Sets `Deno.noColor` in JS runtime. + pub no_color: bool, + /// Sets `Deno.version.deno` in JS runtime. + pub runtime_version: String, + /// Sets `Deno.version.typescript` in JS runtime. + pub ts_version: String, + pub unstable: bool, +} + +impl BootstrapOptions { + pub fn as_json(&self) -> String { + let payload = json!({ + // Shared bootstrap args + "args": self.args, + "applySourceMaps": self.apply_source_maps, + "cpuCount": self.cpu_count, + "debugFlag": self.debug_flag, + "denoVersion": self.runtime_version, + "location": self.location, + "noColor": self.no_color, + "tsVersion": self.ts_version, + "unstableFlag": self.unstable, + // Web worker only + "enableTestingFeaturesFlag": self.enable_testing_features, + // Env values + "pid": std::process::id(), + "ppid": ppid(), + "target": env!("TARGET"), + "v8Version": deno_core::v8_version(), + }); + serde_json::to_string_pretty(&payload).unwrap() + } +}