feat(ext/ffi): add support for buffer arguments (#12335)

This commit adds support for passing buffer arguments across 
FFI boundary.


Co-authored-by: eliassjogreen <eliassjogreen1@gmail.com>
Co-authored-by: Bert Belder <bertbelder@gmail.com>

ext/ffi/00_ffi.js

@@ -4,7 +4,9 @@
 ((window) => {
   const core = window.Deno.core;
   const __bootstrap = window.__bootstrap;
-
+  const {
+    ArrayBuffer,
+  } = window.__bootstrap.primordials;
   class DynamicLibrary {
     #rid;
     symbols = {};
@@ -13,15 +15,40 @@
       this.#rid = core.opSync("op_ffi_load", { path, symbols });
 
       for (const symbol in symbols) {
-        this.symbols[symbol] = symbols[symbol].nonblocking
-          ? (...parameters) =>
-            core.opAsync("op_ffi_call_nonblocking", {
+        const isNonBlocking = symbols[symbol].nonblocking;
+
+        this.symbols[symbol] = (...args) => {
+          const parameters = [];
+          const buffers = [];
+
+          for (const arg of args) {
+            if (
+              arg?.buffer instanceof ArrayBuffer &&
+              arg.byteLength !== undefined
+            ) {
+              parameters.push(buffers.length);
+              buffers.push(arg);
+            } else {
+              parameters.push(arg);
+            }
+          }
+
+          if (isNonBlocking) {
+            return core.opAsync("op_ffi_call_nonblocking", {
               rid: this.#rid,
               symbol,
               parameters,
-            })
-          : (...parameters) =>
-            core.opSync("op_ffi_call", { rid: this.#rid, symbol, parameters });
+              buffers,
+            });
+          } else {
+            return core.opSync("op_ffi_call", {
+              rid: this.#rid,
+              symbol,
+              parameters,
+              buffers,
+            });
+          }
+        };
       }
     }
 

ext/ffi/lib.rs

@@ -11,6 +11,7 @@ use deno_core::Extension;
 use deno_core::OpState;
 use deno_core::Resource;
 use deno_core::ResourceId;
+use deno_core::ZeroCopyBuf;
 use dlopen::raw::Library;
 use libffi::middle::Arg;
 use serde::Deserialize;
@@ -131,6 +132,7 @@ enum NativeType {
   ISize,
   F32,
   F64,
+  Buffer,
 }
 
 impl From<NativeType> for libffi::middle::Type {
@@ -149,6 +151,7 @@ impl From<NativeType> for libffi::middle::Type {
       NativeType::ISize => libffi::middle::Type::isize(),
       NativeType::F32 => libffi::middle::Type::f32(),
       NativeType::F64 => libffi::middle::Type::f64(),
+      NativeType::Buffer => libffi::middle::Type::pointer(),
     }
   }
 }
@@ -168,6 +171,7 @@ union NativeValue {
   isize_value: isize,
   f32_value: f32,
   f64_value: f64,
+  buffer: *const u8,
 }
 
 impl NativeValue {
@@ -210,9 +214,14 @@ impl NativeValue {
       NativeType::F64 => Self {
         f64_value: value_as_f64(value),
       },
+      NativeType::Buffer => unreachable!(),
     }
   }
 
+  fn buffer(ptr: *const u8) -> Self {
+    Self { buffer: ptr }
+  }
+
   unsafe fn as_arg(&self, native_type: NativeType) -> Arg {
     match native_type {
       NativeType::Void => Arg::new(&self.void_value),
@@ -228,6 +237,7 @@ impl NativeValue {
       NativeType::ISize => Arg::new(&self.isize_value),
       NativeType::F32 => Arg::new(&self.f32_value),
       NativeType::F64 => Arg::new(&self.f64_value),
+      NativeType::Buffer => Arg::new(&self.buffer),
     }
   }
 }
@@ -257,6 +267,7 @@ fn value_as_f64(value: Value) -> f64 {
 }
 
 #[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
 struct ForeignFunction {
   parameters: Vec<NativeType>,
   result: NativeType,
@@ -293,20 +304,32 @@ where
   Ok(state.resource_table.add(resource))
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct FfiCallArgs {
   rid: ResourceId,
   symbol: String,
   parameters: Vec<Value>,
+  buffers: Vec<ZeroCopyBuf>,
 }
 
 fn ffi_call(args: FfiCallArgs, symbol: &Symbol) -> Result<Value, AnyError> {
+  let buffers: Vec<&[u8]> =
+    args.buffers.iter().map(|buffer| &buffer[..]).collect();
+
   let native_values = symbol
     .parameter_types
     .iter()
     .zip(args.parameters.into_iter())
-    .map(|(&native_type, value)| NativeValue::new(native_type, value))
+    .map(|(&native_type, value)| {
+      if let NativeType::Buffer = native_type {
+        let idx: usize = value_as_uint(value);
+        let ptr = buffers[idx].as_ptr();
+        NativeValue::buffer(ptr)
+      } else {
+        NativeValue::new(native_type, value)
+      }
+    })
     .collect::<Vec<_>>();
 
   let call_args = symbol
@@ -358,6 +381,7 @@ fn ffi_call(args: FfiCallArgs, symbol: &Symbol) -> Result<Value, AnyError> {
     NativeType::F64 => {
       json!(unsafe { symbol.cif.call::<f64>(symbol.ptr, &call_args) })
     }
+    NativeType::Buffer => unreachable!(),
   })
 }
 

test_ffi/src/lib.rs

@@ -1,3 +1,5 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
 use std::thread::sleep;
 use std::time::Duration;
 
@@ -6,6 +8,13 @@ pub extern "C" fn print_something() {
   println!("something");
 }
 
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn print_buffer(ptr: *const u8, len: usize) {
+  let buf = unsafe { std::slice::from_raw_parts(ptr, len) };
+  println!("{:?}", buf);
+}
+
 #[no_mangle]
 pub extern "C" fn add_u32(a: u32, b: u32) -> u32 {
   a + b
@@ -51,3 +60,10 @@ pub extern "C" fn sleep_blocking(ms: u64) {
   let duration = Duration::from_millis(ms);
   sleep(duration);
 }
+
+#[allow(clippy::not_unsafe_ptr_arg_deref)]
+#[no_mangle]
+pub extern "C" fn nonblocking_buffer(ptr: *const u8, len: usize) {
+  let buf = unsafe { std::slice::from_raw_parts(ptr, len) };
+  assert_eq!(buf, vec![1, 2, 3, 4, 5, 6, 7, 8]);
+}

test_ffi/tests/integration_tests.rs

@@ -38,6 +38,7 @@ fn basic() {
   assert!(output.status.success());
   let expected = "\
     something\n\
+    [1, 2, 3, 4, 5, 6, 7, 8]\n\
     579\n\
     579\n\
     579\n\

test_ffi/tests/test.js

@@ -12,6 +12,7 @@ const libPath = `${targetDir}/${libPrefix}test_ffi.${libSuffix}`;
 const resourcesPre = Deno.resources();
 const dylib = Deno.dlopen(libPath, {
   "print_something": { parameters: [], result: "void" },
+  "print_buffer": { parameters: ["buffer", "usize"], result: "void" },
   "add_u32": { parameters: ["u32", "u32"], result: "u32" },
   "add_i32": { parameters: ["i32", "i32"], result: "i32" },
   "add_u64": { parameters: ["u64", "u64"], result: "u64" },
@@ -21,9 +22,16 @@ const dylib = Deno.dlopen(libPath, {
   "add_f32": { parameters: ["f32", "f32"], result: "f32" },
   "add_f64": { parameters: ["f64", "f64"], result: "f64" },
   "sleep_blocking": { parameters: ["u64"], result: "void", nonblocking: true },
+  "nonblocking_buffer": {
+    parameters: ["buffer", "usize"],
+    result: "void",
+    nonblocking: true,
+  },
 });
 
 dylib.symbols.print_something();
+const buffer = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+dylib.symbols.print_buffer(buffer, buffer.length);
 console.log(dylib.symbols.add_u32(123, 456));
 console.log(dylib.symbols.add_i32(123, 456));
 console.log(dylib.symbols.add_u64(123, 456));
@@ -34,6 +42,30 @@ console.log(dylib.symbols.add_f32(123.123, 456.789));
 console.log(dylib.symbols.add_f64(123.123, 456.789));
 
 // Test non blocking calls
+
+function deferred() {
+  let methods;
+  const promise = new Promise((resolve, reject) => {
+    methods = {
+      async resolve(value) {
+        await value;
+        resolve(value);
+      },
+      reject(reason) {
+        reject(reason);
+      },
+    };
+  });
+  return Object.assign(promise, methods);
+}
+
+const promise = deferred();
+const buffer2 = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]);
+dylib.symbols.nonblocking_buffer(buffer2, buffer2.length).then(() => {
+  promise.resolve();
+});
+await promise;
+
 const start = performance.now();
 dylib.symbols.sleep_blocking(100).then(() => {
   console.log("After");