Revert "Remove unstable native plugins (#10908)"

This reverts commit 7dd4090c2a.

Cargo.lock

@@ -3796,6 +3796,16 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "test_plugin"
+version = "0.0.1"
+dependencies = [
+ "deno_core",
+ "futures",
+ "serde",
+ "test_util",
+]
+
 [[package]]
 name = "test_util"
 version = "0.1.0"

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "cli",
   "core",
   "runtime",
+  "test_plugin",
   "test_util",
   "extensions/broadcast_channel",
   "extensions/console",

cli/dts/lib.deno.unstable.d.ts

@@ -129,6 +129,37 @@ declare namespace Deno {
     speed: number | undefined;
   }
 
+  /** **UNSTABLE**: new API, yet to be vetted.
+   *
+   * Open and initialize a plugin.
+   *
+   * ```ts
+   * import { assert } from "https://deno.land/std/testing/asserts.ts";
+   * const rid = Deno.openPlugin("./path/to/some/plugin.so");
+   *
+   * // The Deno.core namespace is needed to interact with plugins, but this is
+   * // internal so we use ts-ignore to skip type checking these calls.
+   * // @ts-ignore
+   * const { op_test_sync, op_test_async } = Deno.core.ops();
+   *
+   * assert(op_test_sync);
+   * assert(op_test_async);
+   *
+   * // @ts-ignore
+   * const result = Deno.core.opSync("op_test_sync");
+   *
+   * // @ts-ignore
+   * const result = await Deno.core.opAsync("op_test_sync");
+   * ```
+   *
+   * Requires `allow-plugin` permission.
+   *
+   * The plugin system is not stable and will change in the future, hence the
+   * lack of docs. For now take a look at the example
+   * https://github.com/denoland/deno/tree/main/test_plugin
+   */
+  export function openPlugin(filename: string): number;
+
   /** The log category for a diagnostic message. */
   export enum DiagnosticCategory {
     Warning = 0,

cli/tests/integration/lsp_tests.rs

@@ -470,7 +470,7 @@ fn lsp_hover_unstable_enabled() {
         "uri": "file:///a/file.ts",
         "languageId": "typescript",
         "version": 1,
-        "text": "console.log(Deno.ppid);\n"
+        "text": "console.log(Deno.openPlugin);\n"
       }
     }),
   );
@@ -495,9 +495,9 @@ fn lsp_hover_unstable_enabled() {
       "contents":[
         {
           "language":"typescript",
-          "value":"const Deno.ppid: number"
+          "value":"function Deno.openPlugin(filename: string): number"
         },
-        "The pid of the current process's parent."
+        "**UNSTABLE**: new API, yet to be vetted.\n\nOpen and initialize a plugin.\n\n```ts\nimport { assert } from \"https://deno.land/std/testing/asserts.ts\";\nconst rid = Deno.openPlugin(\"./path/to/some/plugin.so\");\n\n// The Deno.core namespace is needed to interact with plugins, but this is\n// internal so we use ts-ignore to skip type checking these calls.\n// @ts-ignore\nconst { op_test_sync, op_test_async } = Deno.core.ops();\n\nassert(op_test_sync);\nassert(op_test_async);\n\n// @ts-ignore\nconst result = Deno.core.opSync(\"op_test_sync\");\n\n// @ts-ignore\nconst result = await Deno.core.opAsync(\"op_test_sync\");\n```\n\nRequires `allow-plugin` permission.\n\nThe plugin system is not stable and will change in the future, hence the\nlack of docs. For now take a look at the example\nhttps://github.com/denoland/deno/tree/main/test_plugin"
       ],
       "range":{
         "start":{
@@ -506,7 +506,7 @@ fn lsp_hover_unstable_enabled() {
         },
         "end":{
           "line":0,
-          "character":21
+          "character":27
         }
       }
     }))

runtime/js/40_plugins.js

@@ -0,0 +1,16 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+"use strict";
+
+((window) => {
+  const core = window.Deno.core;
+
+  function openPlugin(filename) {
+    const rid = core.opSync("op_open_plugin", filename);
+    core.syncOpsCache();
+    return rid;
+  }
+
+  window.__bootstrap.plugins = {
+    openPlugin,
+  };
+})(this);

runtime/js/90_deno_ns.js

@@ -109,6 +109,7 @@
     Signal: __bootstrap.signals.Signal,
     SignalStream: __bootstrap.signals.SignalStream,
     emit: __bootstrap.compilerApi.emit,
+    openPlugin: __bootstrap.plugins.openPlugin,
     kill: __bootstrap.process.kill,
     setRaw: __bootstrap.tty.setRaw,
     consoleSize: __bootstrap.tty.consoleSize,

runtime/ops/mod.rs

@@ -5,6 +5,7 @@ pub mod fs_events;
 pub mod io;
 pub mod os;
 pub mod permissions;
+pub mod plugin;
 pub mod process;
 pub mod runtime;
 pub mod signal;

runtime/ops/plugin.rs

@@ -0,0 +1,86 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+use crate::permissions::Permissions;
+use deno_core::error::AnyError;
+use deno_core::op_sync;
+use deno_core::Extension;
+use deno_core::OpState;
+use deno_core::Resource;
+use deno_core::ResourceId;
+use dlopen::symbor::Library;
+use log::debug;
+use std::borrow::Cow;
+use std::mem;
+use std::path::PathBuf;
+use std::rc::Rc;
+
+/// A default `init` function for plugins which mimics the way the internal
+/// extensions are initalized. Plugins currently do not support all extension
+/// features and are most likely not going to in the future. Currently only
+/// `init_state` and `init_ops` are supported while `init_middleware` and `init_js`
+/// are not. Currently the `PluginResource` does not support being closed due to
+/// certain risks in unloading the dynamic library without unloading dependent
+/// functions and resources.
+pub type InitFn = fn() -> Extension;
+
+pub fn init() -> Extension {
+  Extension::builder()
+    .ops(vec![("op_open_plugin", op_sync(op_open_plugin))])
+    .build()
+}
+
+pub fn op_open_plugin(
+  state: &mut OpState,
+  filename: String,
+  _: (),
+) -> Result<ResourceId, AnyError> {
+  let filename = PathBuf::from(&filename);
+
+  super::check_unstable(state, "Deno.openPlugin");
+  let permissions = state.borrow_mut::<Permissions>();
+  permissions.plugin.check()?;
+
+  debug!("Loading Plugin: {:#?}", filename);
+  let plugin_lib = Library::open(filename).map(Rc::new)?;
+  let plugin_resource = PluginResource::new(&plugin_lib);
+
+  // Forgets the plugin_lib value to prevent segfaults when the process exits
+  mem::forget(plugin_lib);
+
+  let init = *unsafe { plugin_resource.0.symbol::<InitFn>("init") }?;
+  let rid = state.resource_table.add(plugin_resource);
+  let mut extension = init();
+
+  if !extension.init_js().is_empty() {
+    panic!("Plugins do not support loading js");
+  }
+
+  if extension.init_middleware().is_some() {
+    panic!("Plugins do not support middleware");
+  }
+
+  extension.init_state(state)?;
+  let ops = extension.init_ops().unwrap_or_default();
+  for (name, opfn) in ops {
+    state.op_table.register_op(name, opfn);
+  }
+
+  Ok(rid)
+}
+
+struct PluginResource(Rc<Library>);
+
+impl Resource for PluginResource {
+  fn name(&self) -> Cow<str> {
+    "plugin".into()
+  }
+
+  fn close(self: Rc<Self>) {
+    unimplemented!();
+  }
+}
+
+impl PluginResource {
+  fn new(lib: &Rc<Library>) -> Self {
+    Self(lib.clone())
+  }
+}

runtime/web_worker.rs

@@ -335,6 +335,7 @@ impl WebWorker {
         deno_net::init::<Permissions>(options.unstable),
         ops::os::init(),
         ops::permissions::init(),
+        ops::plugin::init(),
         ops::process::init(),
         ops::signal::init(),
         ops::tty::init(),

runtime/worker.rs

@@ -126,6 +126,7 @@ impl MainWorker {
       deno_net::init::<Permissions>(options.unstable),
       ops::os::init(),
       ops::permissions::init(),
+      ops::plugin::init(),
       ops::process::init(),
       ops::signal::init(),
       ops::tty::init(),

test_plugin/Cargo.toml

@@ -0,0 +1,19 @@
+# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+[package]
+name = "test_plugin"
+version = "0.0.1"
+authors = ["the deno authors"]
+edition = "2018"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+deno_core = { path = "../core" }
+futures = "0.3.15"
+serde = "1"
+
+[dev-dependencies]
+test_util = { path = "../test_util" }

test_plugin/README.md

@@ -0,0 +1,9 @@
+# `test_plugin` crate
+
+## To run this test manually
+
+```
+cd test_plugin
+
+../target/debug/deno run --unstable --allow-plugin tests/test.js debug
+```

test_plugin/src/lib.rs

@@ -0,0 +1,114 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use std::borrow::Cow;
+use std::cell::RefCell;
+use std::rc::Rc;
+
+use deno_core::error::bad_resource_id;
+use deno_core::error::AnyError;
+use deno_core::op_async;
+use deno_core::op_sync;
+use deno_core::Extension;
+use deno_core::OpState;
+use deno_core::Resource;
+use deno_core::ResourceId;
+use deno_core::ZeroCopyBuf;
+use serde::Deserialize;
+
+#[no_mangle]
+pub fn init() -> Extension {
+  Extension::builder()
+    .ops(vec![
+      ("op_test_sync", op_sync(op_test_sync)),
+      ("op_test_async", op_async(op_test_async)),
+      (
+        "op_test_resource_table_add",
+        op_sync(op_test_resource_table_add),
+      ),
+      (
+        "op_test_resource_table_get",
+        op_sync(op_test_resource_table_get),
+      ),
+    ])
+    .build()
+}
+
+#[derive(Debug, Deserialize)]
+struct TestArgs {
+  val: String,
+}
+
+fn op_test_sync(
+  _state: &mut OpState,
+  args: TestArgs,
+  zero_copy: Option<ZeroCopyBuf>,
+) -> Result<String, AnyError> {
+  println!("Hello from sync plugin op.");
+
+  println!("args: {:?}", args);
+
+  if let Some(buf) = zero_copy {
+    let buf_str = std::str::from_utf8(&buf[..])?;
+    println!("zero_copy: {}", buf_str);
+  }
+
+  Ok("test".to_string())
+}
+
+async fn op_test_async(
+  _state: Rc<RefCell<OpState>>,
+  args: TestArgs,
+  zero_copy: Option<ZeroCopyBuf>,
+) -> Result<String, AnyError> {
+  println!("Hello from async plugin op.");
+
+  println!("args: {:?}", args);
+
+  if let Some(buf) = zero_copy {
+    let buf_str = std::str::from_utf8(&buf[..])?;
+    println!("zero_copy: {}", buf_str);
+  }
+
+  let (tx, rx) = futures::channel::oneshot::channel::<Result<(), ()>>();
+  std::thread::spawn(move || {
+    std::thread::sleep(std::time::Duration::from_secs(1));
+    tx.send(Ok(())).unwrap();
+  });
+  assert!(rx.await.is_ok());
+
+  Ok("test".to_string())
+}
+
+struct TestResource(String);
+impl Resource for TestResource {
+  fn name(&self) -> Cow<str> {
+    "TestResource".into()
+  }
+}
+
+fn op_test_resource_table_add(
+  state: &mut OpState,
+  text: String,
+  _: (),
+) -> Result<u32, AnyError> {
+  println!("Hello from resource_table.add plugin op.");
+
+  Ok(state.resource_table.add(TestResource(text)))
+}
+
+fn op_test_resource_table_get(
+  state: &mut OpState,
+  rid: ResourceId,
+  _: (),
+) -> Result<String, AnyError> {
+  println!("Hello from resource_table.get plugin op.");
+
+  Ok(
+    state
+      .resource_table
+      .get::<TestResource>(rid)
+      .ok_or_else(bad_resource_id)?
+      .0
+      .clone(),
+  )
+}

test_plugin/tests/integration_tests.rs

@@ -0,0 +1,58 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use std::process::Command;
+use test_util::deno_cmd;
+
+#[cfg(debug_assertions)]
+const BUILD_VARIANT: &str = "debug";
+
+#[cfg(not(debug_assertions))]
+const BUILD_VARIANT: &str = "release";
+
+#[test]
+fn basic() {
+  let mut build_plugin_base = Command::new("cargo");
+  let mut build_plugin =
+    build_plugin_base.arg("build").arg("-p").arg("test_plugin");
+  if BUILD_VARIANT == "release" {
+    build_plugin = build_plugin.arg("--release");
+  }
+  let build_plugin_output = build_plugin.output().unwrap();
+  assert!(build_plugin_output.status.success());
+  let output = deno_cmd()
+    .arg("run")
+    .arg("--allow-plugin")
+    .arg("--unstable")
+    .arg("tests/test.js")
+    .arg(BUILD_VARIANT)
+    .output()
+    .unwrap();
+  let stdout = std::str::from_utf8(&output.stdout).unwrap();
+  let stderr = std::str::from_utf8(&output.stderr).unwrap();
+  if !output.status.success() {
+    println!("stdout {}", stdout);
+    println!("stderr {}", stderr);
+  }
+  println!("{:?}", output.status);
+  assert!(output.status.success());
+  let expected = "\
+    Plugin rid: 3\n\
+    Hello from sync plugin op.\n\
+    args: TestArgs { val: \"1\" }\n\
+    zero_copy: test\n\
+    op_test_sync returned: test\n\
+    Hello from async plugin op.\n\
+    args: TestArgs { val: \"1\" }\n\
+    zero_copy: 123\n\
+    op_test_async returned: test\n\
+    Hello from resource_table.add plugin op.\n\
+    TestResource rid: 4\n\
+    Hello from resource_table.get plugin op.\n\
+    TestResource get value: hello plugin!\n\
+    Hello from sync plugin op.\n\
+    args: TestArgs { val: \"1\" }\n\
+    Ops completed count is correct!\n\
+    Ops dispatched count is correct!\n";
+  assert_eq!(stdout, expected);
+  assert_eq!(stderr, "");
+}

test_plugin/tests/test.js

@@ -0,0 +1,135 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+// deno-lint-ignore-file
+
+const filenameBase = "test_plugin";
+
+let filenameSuffix = ".so";
+let filenamePrefix = "lib";
+
+if (Deno.build.os === "windows") {
+  filenameSuffix = ".dll";
+  filenamePrefix = "";
+} else if (Deno.build.os === "darwin") {
+  filenameSuffix = ".dylib";
+}
+
+const filename = `../target/${
+  Deno.args[0]
+}/${filenamePrefix}${filenameBase}${filenameSuffix}`;
+
+const resourcesPre = Deno.resources();
+
+const pluginRid = Deno.openPlugin(filename);
+console.log(`Plugin rid: ${pluginRid}`);
+
+const {
+  op_test_sync,
+  op_test_async,
+  op_test_resource_table_add,
+  op_test_resource_table_get,
+} = Deno.core.ops();
+
+if (
+  op_test_sync === null ||
+  op_test_async === null ||
+  op_test_resource_table_add === null ||
+  op_test_resource_table_get === null
+) {
+  throw new Error("Not all expected ops were registered");
+}
+
+function runTestSync() {
+  const result = Deno.core.opSync(
+    "op_test_sync",
+    { val: "1" },
+    new Uint8Array([116, 101, 115, 116]),
+  );
+
+  console.log(`op_test_sync returned: ${result}`);
+
+  if (result !== "test") {
+    throw new Error("op_test_sync returned an unexpected value!");
+  }
+}
+
+async function runTestAsync() {
+  const promise = Deno.core.opAsync(
+    "op_test_async",
+    { val: "1" },
+    new Uint8Array([49, 50, 51]),
+  );
+
+  if (!(promise instanceof Promise)) {
+    throw new Error("Expected promise!");
+  }
+
+  const result = await promise;
+  console.log(`op_test_async returned: ${result}`);
+
+  if (result !== "test") {
+    throw new Error("op_test_async promise resolved to an unexpected value!");
+  }
+}
+
+function runTestResourceTable() {
+  const expect = "hello plugin!";
+
+  const testRid = Deno.core.opSync("op_test_resource_table_add", expect);
+  console.log(`TestResource rid: ${testRid}`);
+
+  if (testRid === null || Deno.resources()[testRid] !== "TestResource") {
+    throw new Error("TestResource was not found!");
+  }
+
+  const testValue = Deno.core.opSync("op_test_resource_table_get", testRid);
+  console.log(`TestResource get value: ${testValue}`);
+
+  if (testValue !== expect) {
+    throw new Error("Did not get correct resource value!");
+  }
+
+  Deno.close(testRid);
+}
+
+function runTestOpCount() {
+  const start = Deno.metrics();
+
+  Deno.core.opSync("op_test_sync", { val: "1" });
+
+  const end = Deno.metrics();
+
+  if (end.opsCompleted - start.opsCompleted !== 1) {
+    throw new Error("The opsCompleted metric is not correct!");
+  }
+  console.log("Ops completed count is correct!");
+
+  if (end.opsDispatched - start.opsDispatched !== 1) {
+    throw new Error("The opsDispatched metric is not correct!");
+  }
+  console.log("Ops dispatched count is correct!");
+}
+
+function runTestPluginClose() {
+  // Closing does not yet work
+  Deno.close(pluginRid);
+
+  const resourcesPost = Deno.resources();
+
+  const preStr = JSON.stringify(resourcesPre, null, 2);
+  const postStr = JSON.stringify(resourcesPost, null, 2);
+  if (preStr !== postStr) {
+    throw new Error(
+      `Difference in open resources before openPlugin and after Plugin.close():
+Before: ${preStr}
+After: ${postStr}`,
+    );
+  }
+  console.log("Correct number of resources");
+}
+
+runTestSync();
+await runTestAsync();
+runTestResourceTable();
+
+runTestOpCount();
+// runTestPluginClose();