From d359789c529d3c7b5fab5471309eaa4b75fc0bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Thu, 16 Apr 2020 23:40:29 +0200 Subject: [PATCH] feat: support Deno namespace in Worker API (#4784) --- cli/compilers/compiler_worker.rs | 2 +- cli/js/compiler.ts | 4 +-- cli/js/lib.deno.shared_globals.d.ts | 40 +++++++++++++++++++++++++++ cli/js/lib.deno.worker.d.ts | 2 ++ cli/js/ops/worker_host.ts | 2 ++ cli/js/runtime_worker.ts | 32 +++++++++++++++++++-- cli/js/web/workers.ts | 4 +++ cli/ops/worker_host.rs | 43 +++++++++++++++++++++-------- cli/tests/subdir/deno_worker.ts | 7 +++++ cli/tests/subdir/non_deno_worker.js | 7 +++++ cli/tests/workers_test.out | 5 ++-- cli/tests/workers_test.ts | 33 ++++++++++++++++++++++ cli/web_worker.rs | 30 ++++++++++++++++++-- 13 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 cli/tests/subdir/deno_worker.ts create mode 100644 cli/tests/subdir/non_deno_worker.js diff --git a/cli/compilers/compiler_worker.rs b/cli/compilers/compiler_worker.rs index a1d2dc71c0..aa84c8695e 100644 --- a/cli/compilers/compiler_worker.rs +++ b/cli/compilers/compiler_worker.rs @@ -30,7 +30,7 @@ pub struct CompilerWorker(WebWorker); impl CompilerWorker { pub fn new(name: String, startup_data: StartupData, state: State) -> Self { let state_ = state.clone(); - let mut worker = WebWorker::new(name, startup_data, state_); + let mut worker = WebWorker::new(name, startup_data, state_, false); { let isolate = &mut worker.isolate; ops::compiler::init(isolate, &state); diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts index b3cd3a481e..91653a8e4e 100644 --- a/cli/js/compiler.ts +++ b/cli/js/compiler.ts @@ -391,12 +391,12 @@ async function wasmCompilerOnMessage({ } function bootstrapTsCompilerRuntime(): void { - bootstrapWorkerRuntime("TS"); + bootstrapWorkerRuntime("TS", false); globalThis.onmessage = tsCompilerOnMessage; } function bootstrapWasmCompilerRuntime(): void { - bootstrapWorkerRuntime("WASM"); + bootstrapWorkerRuntime("WASM", false); globalThis.onmessage = wasmCompilerOnMessage; } diff --git a/cli/js/lib.deno.shared_globals.d.ts b/cli/js/lib.deno.shared_globals.d.ts index fd9f3691d6..ef450c2016 100644 --- a/cli/js/lib.deno.shared_globals.d.ts +++ b/cli/js/lib.deno.shared_globals.d.ts @@ -1080,6 +1080,46 @@ declare class Worker extends EventTarget { options?: { type?: "classic" | "module"; name?: string; + /** UNSTABLE: New API. Expect many changes; most likely this + * field will be made into an object for more granular + * configuration of worker thread (permissions, import map, etc.). + * + * Set to `true` to make `Deno` namespace and all of its methods + * available to worker thread. + * + * Currently worker inherits permissions from main thread (permissions + * given using `--allow-*` flags). + * Configurable permissions are on the roadmap to be implemented. + * + * Example: + * // mod.ts + * const worker = new Worker("./deno_worker.ts", { type: "module", deno: true }); + * worker.postMessage({ cmd: "readFile", fileName: "./log.txt" }); + * + * // deno_worker.ts + * + * + * self.onmessage = async function (e) { + * const { cmd, fileName } = e.data; + * if (cmd !== "readFile") { + * throw new Error("Invalid command"); + * } + * const buf = await Deno.readFile(fileName); + * const fileContents = new TextDecoder().decode(buf); + * console.log(fileContents); + * } + * + * // log.txt + * hello world + * hello world 2 + * + * // run program + * $ deno run --allow-read mod.ts + * hello world + * hello world2 + * + */ + deno?: boolean; } ); postMessage(message: any, transfer: ArrayBuffer[]): void; diff --git a/cli/js/lib.deno.worker.d.ts b/cli/js/lib.deno.worker.d.ts index a1e83af485..bba5f33219 100644 --- a/cli/js/lib.deno.worker.d.ts +++ b/cli/js/lib.deno.worker.d.ts @@ -3,6 +3,7 @@ /* eslint-disable @typescript-eslint/no-unused-vars, @typescript-eslint/no-explicit-any */ /// +/// /// /// @@ -15,6 +16,7 @@ declare interface DedicatedWorkerGlobalScope { name: typeof __workerMain.name; close: typeof __workerMain.close; postMessage: typeof __workerMain.postMessage; + Deno: typeof Deno; } declare const self: DedicatedWorkerGlobalScope & typeof globalThis; diff --git a/cli/js/ops/worker_host.ts b/cli/js/ops/worker_host.ts index d483a5313a..11e268152b 100644 --- a/cli/js/ops/worker_host.ts +++ b/cli/js/ops/worker_host.ts @@ -6,6 +6,7 @@ export function createWorker( specifier: string, hasSourceCode: boolean, sourceCode: string, + useDenoNamespace: boolean, name?: string ): { id: number } { return sendSync("op_create_worker", { @@ -13,6 +14,7 @@ export function createWorker( hasSourceCode, sourceCode, name, + useDenoNamespace, }); } diff --git a/cli/js/runtime_worker.ts b/cli/js/runtime_worker.ts index 1e7a9a67da..60c845a184 100644 --- a/cli/js/runtime_worker.ts +++ b/cli/js/runtime_worker.ts @@ -17,12 +17,23 @@ import { eventTargetProperties, setEventTargetData, } from "./globals.ts"; +import * as Deno from "./deno.ts"; import * as webWorkerOps from "./ops/web_worker.ts"; import { LocationImpl } from "./web/location.ts"; import { log, assert, immutableDefine } from "./util.ts"; import { MessageEvent, ErrorEvent } from "./web/workers.ts"; import { TextEncoder } from "./web/text_encoding.ts"; import * as runtime from "./runtime.ts"; +import { internalObject } from "./internals.ts"; +import { symbols } from "./symbols.ts"; +import { setSignals } from "./signals.ts"; + +// FIXME(bartlomieju): duplicated in `runtime_main.ts` +// TODO: factor out `Deno` global assignment to separate function +// Add internal object to Deno object. +// This is not exposed as part of the Deno types. +// @ts-ignore +Deno[symbols.internal] = internalObject; const encoder = new TextEncoder(); @@ -109,6 +120,7 @@ export const workerRuntimeGlobalProperties = { export function bootstrapWorkerRuntime( name: string, + useDenoNamespace: boolean, internalName?: string ): void { if (hasBootstrapped) { @@ -128,7 +140,21 @@ export function bootstrapWorkerRuntime( immutableDefine(globalThis, "location", location); Object.freeze(globalThis.location); - // globalThis.Deno is not available in worker scope - delete globalThis.Deno; - assert(globalThis.Deno === undefined); + if (useDenoNamespace) { + Object.defineProperties(Deno, { + pid: readOnly(s.pid), + noColor: readOnly(s.noColor), + args: readOnly(Object.freeze(s.args)), + }); + // Setup `Deno` global - we're actually overriding already + // existing global `Deno` with `Deno` namespace from "./deno.ts". + immutableDefine(globalThis, "Deno", Deno); + Object.freeze(globalThis.Deno); + Object.freeze(globalThis.Deno.core); + Object.freeze(globalThis.Deno.core.sharedQueue); + setSignals(); + } else { + delete globalThis.Deno; + assert(globalThis.Deno === undefined); + } } diff --git a/cli/js/web/workers.ts b/cli/js/web/workers.ts index 0b7d4f4b6a..6fcab3fe3b 100644 --- a/cli/js/web/workers.ts +++ b/cli/js/web/workers.ts @@ -105,6 +105,7 @@ export interface Worker { export interface WorkerOptions { type?: "classic" | "module"; name?: string; + deno?: boolean; } export class WorkerImpl extends EventTarget implements Worker { @@ -146,10 +147,13 @@ export class WorkerImpl extends EventTarget implements Worker { } */ + const useDenoNamespace = options ? !!options.deno : false; + const { id } = createWorker( specifier, hasSourceCode, sourceCode, + useDenoNamespace, options?.name ); this.#id = id; diff --git a/cli/ops/worker_host.rs b/cli/ops/worker_host.rs index dafd969b2e..a4db5e27fb 100644 --- a/cli/ops/worker_host.rs +++ b/cli/ops/worker_host.rs @@ -3,6 +3,7 @@ use super::dispatch_json::{Deserialize, JsonOp, Value}; use crate::fmt_errors::JSError; use crate::global_state::GlobalState; use crate::op_error::OpError; +use crate::ops::io::get_stdio; use crate::permissions::DenoPermissions; use crate::startup_data; use crate::state::State; @@ -37,17 +38,31 @@ fn create_web_worker( global_state: GlobalState, permissions: DenoPermissions, specifier: ModuleSpecifier, + has_deno_namespace: bool, ) -> Result { let state = State::new_for_worker(global_state, Some(permissions), specifier)?; - let mut worker = - WebWorker::new(name.to_string(), startup_data::deno_isolate_init(), state); + if has_deno_namespace { + let mut s = state.borrow_mut(); + let (stdin, stdout, stderr) = get_stdio(); + s.resource_table.add("stdin", Box::new(stdin)); + s.resource_table.add("stdout", Box::new(stdout)); + s.resource_table.add("stderr", Box::new(stderr)); + } + + let mut worker = WebWorker::new( + name.clone(), + startup_data::deno_isolate_init(), + state, + has_deno_namespace, + ); + // Instead of using name for log we use `worker-${id}` because // WebWorkers can have empty string as name. let script = format!( - "bootstrapWorkerRuntime(\"{}\", \"worker-{}\")", - name, worker_id + "bootstrapWorkerRuntime(\"{}\", {}, \"worker-{}\")", + name, worker.has_deno_namespace, worker_id ); worker.execute(&script)?; @@ -61,8 +76,8 @@ fn run_worker_thread( global_state: GlobalState, permissions: DenoPermissions, specifier: ModuleSpecifier, - has_source_code: bool, - source_code: String, + has_deno_namespace: bool, + maybe_source_code: Option, ) -> Result<(JoinHandle<()>, WebWorkerHandle), ErrBox> { let (handle_sender, handle_receiver) = std::sync::mpsc::sync_channel::>(1); @@ -80,6 +95,7 @@ fn run_worker_thread( global_state, permissions, specifier.clone(), + has_deno_namespace, ); if let Err(err) = result { @@ -108,7 +124,7 @@ fn run_worker_thread( // TODO: run with using select with terminate // Execute provided source code immediately - let result = if has_source_code { + let result = if let Some(source_code) = maybe_source_code { worker.execute(&source_code) } else { // TODO(bartlomieju): add "type": "classic", ie. ability to load @@ -146,6 +162,7 @@ struct CreateWorkerArgs { specifier: String, has_source_code: bool, source_code: String, + use_deno_namespace: bool, } /// Create worker as the host @@ -157,9 +174,13 @@ fn op_create_worker( let args: CreateWorkerArgs = serde_json::from_value(args)?; let specifier = args.specifier.clone(); - let has_source_code = args.has_source_code; - let source_code = args.source_code.clone(); + let maybe_source_code = if args.has_source_code { + Some(args.source_code.clone()) + } else { + None + }; let args_name = args.name; + let use_deno_namespace = args.use_deno_namespace; let parent_state = state.clone(); let mut state = state.borrow_mut(); let global_state = state.global_state.clone(); @@ -179,8 +200,8 @@ fn op_create_worker( global_state, permissions, module_specifier, - has_source_code, - source_code, + use_deno_namespace, + maybe_source_code, ) .map_err(|e| OpError::other(e.to_string()))?; // At this point all interactions with worker happen using thread diff --git a/cli/tests/subdir/deno_worker.ts b/cli/tests/subdir/deno_worker.ts new file mode 100644 index 0000000000..6a57c47f0f --- /dev/null +++ b/cli/tests/subdir/deno_worker.ts @@ -0,0 +1,7 @@ +onmessage = function (e): void { + if (typeof self.Deno === "undefined") { + throw new Error("Deno namespace not available in worker"); + } + + postMessage(e.data); +}; diff --git a/cli/tests/subdir/non_deno_worker.js b/cli/tests/subdir/non_deno_worker.js new file mode 100644 index 0000000000..773721560d --- /dev/null +++ b/cli/tests/subdir/non_deno_worker.js @@ -0,0 +1,7 @@ +onmessage = function (e) { + if (typeof self.Deno !== "undefined") { + throw new Error("Deno namespace unexpectedly available in worker"); + } + + postMessage(e.data); +}; diff --git a/cli/tests/workers_test.out b/cli/tests/workers_test.out index d3c25d3ddc..9032262834 100644 --- a/cli/tests/workers_test.out +++ b/cli/tests/workers_test.out @@ -1,4 +1,4 @@ -running 8 tests +running 9 tests test worker terminate ... ok [WILDCARD] test worker nested ... ok [WILDCARD] test worker throws when executing ... ok [WILDCARD] @@ -7,5 +7,6 @@ test worker terminate busy loop ... ok [WILDCARD] test worker race condition ... ok [WILDCARD] test worker is event listener ... ok [WILDCARD] test worker scope is event listener ... ok [WILDCARD] +test worker with Deno namespace ... ok [WILDCARD] -test result: ok. 8 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD] +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD] diff --git a/cli/tests/workers_test.ts b/cli/tests/workers_test.ts index c51adcf757..c4b93db3c5 100644 --- a/cli/tests/workers_test.ts +++ b/cli/tests/workers_test.ts @@ -257,3 +257,36 @@ Deno.test({ worker.terminate(); }, }); + +Deno.test({ + name: "worker with Deno namespace", + fn: async function (): Promise { + const promise = createResolvable(); + const promise2 = createResolvable(); + + const regularWorker = new Worker("../tests/subdir/non_deno_worker.js", { + type: "module", + }); + const denoWorker = new Worker("../tests/subdir/deno_worker.ts", { + type: "module", + deno: true, + }); + + regularWorker.onmessage = (e): void => { + assertEquals(e.data, "Hello World"); + regularWorker.terminate(); + promise.resolve(); + }; + + denoWorker.onmessage = (e): void => { + assertEquals(e.data, "Hello World"); + denoWorker.terminate(); + promise2.resolve(); + }; + + regularWorker.postMessage("Hello World"); + await promise; + denoWorker.postMessage("Hello World"); + await promise2; + }, +}); diff --git a/cli/web_worker.rs b/cli/web_worker.rs index 795409175d..24318fc59e 100644 --- a/cli/web_worker.rs +++ b/cli/web_worker.rs @@ -77,10 +77,16 @@ pub struct WebWorker { event_loop_idle: bool, terminate_rx: mpsc::Receiver<()>, handle: WebWorkerHandle, + pub has_deno_namespace: bool, } impl WebWorker { - pub fn new(name: String, startup_data: StartupData, state: State) -> Self { + pub fn new( + name: String, + startup_data: StartupData, + state: State, + has_deno_namespace: bool, + ) -> Self { let state_ = state.clone(); let mut worker = Worker::new(name, startup_data, state_); @@ -105,6 +111,7 @@ impl WebWorker { event_loop_idle: false, terminate_rx, handle, + has_deno_namespace, }; let handle = web_worker.thread_safe_handle(); @@ -124,6 +131,22 @@ impl WebWorker { ops::errors::init(isolate, &state); ops::timers::init(isolate, &state); ops::fetch::init(isolate, &state); + + if has_deno_namespace { + let op_registry = isolate.op_registry.clone(); + ops::runtime_compiler::init(isolate, &state); + ops::fs::init(isolate, &state); + ops::fs_events::init(isolate, &state); + ops::plugins::init(isolate, &state, op_registry); + ops::net::init(isolate, &state); + ops::tls::init(isolate, &state); + ops::os::init(isolate, &state); + ops::permissions::init(isolate, &state); + ops::process::init(isolate, &state); + ops::random::init(isolate, &state); + ops::signal::init(isolate, &state); + ops::tty::init(isolate, &state); + } } web_worker @@ -238,8 +261,11 @@ mod tests { "TEST".to_string(), startup_data::deno_isolate_init(), state, + false, ); - worker.execute("bootstrapWorkerRuntime(\"TEST\")").unwrap(); + worker + .execute("bootstrapWorkerRuntime(\"TEST\", false)") + .unwrap(); worker } #[test]