perf(ext/ffi): use fast api calls for 64bit return types (#15313)

ext/ffi/00_ffi.js

@@ -7,9 +7,17 @@
   const {
     BigInt,
     ObjectDefineProperty,
+    ArrayPrototypeMap,
+    Number,
+    NumberIsSafeInteger,
+    ArrayPrototypeJoin,
     ObjectPrototypeIsPrototypeOf,
     PromisePrototypeThen,
     TypeError,
+    Int32Array,
+    Uint32Array,
+    BigInt64Array,
+    Function,
   } = window.__bootstrap.primordials;
 
   function unpackU64(returnValue) {
@@ -209,6 +217,10 @@
       type === "usize" || type === "isize";
   }
 
+  function isI64(type) {
+    return type === "i64" || type === "isize";
+  }
+
   class UnsafeCallback {
     #refcount;
     #rid;
@@ -293,11 +305,11 @@
           );
           continue;
         }
+        const resultType = symbols[symbol].result;
+        const needsUnpacking = isReturnedAsBigInt(resultType);
 
         const isNonBlocking = symbols[symbol].nonblocking;
         if (isNonBlocking) {
-          const resultType = symbols[symbol].result;
-          const needsUnpacking = isReturnedAsBigInt(resultType);
           ObjectDefineProperty(
             this.symbols,
             symbol,
@@ -326,6 +338,38 @@
             },
           );
         }
+
+        if (needsUnpacking && !isNonBlocking) {
+          const call = this.symbols[symbol];
+          const parameters = symbols[symbol].parameters;
+          const vi = new Int32Array(2);
+          const vui = new Uint32Array(vi.buffer);
+          const b = new BigInt64Array(vi.buffer);
+
+          const params = ArrayPrototypeJoin(
+            ArrayPrototypeMap(parameters, (_, index) => `p${index}`),
+            ", ",
+          );
+          // Make sure V8 has no excuse to not optimize this function.
+          this.symbols[symbol] = new Function(
+            "vi",
+            "vui",
+            "b",
+            "call",
+            "NumberIsSafeInteger",
+            "Number",
+            `return function (${params}) {
+            call(${params}${parameters.length > 0 ? ", " : ""}vi);
+            ${
+              isI64(resultType)
+                ? `const n1 = Number(b[0])`
+                : `const n1 = vui[0] + 2 ** 32 * vui[1]` // Faster path for u64
+            };
+            if (NumberIsSafeInteger(n1)) return n1;
+            return b[0];
+          }`,
+          )(vi, vui, b, call, NumberIsSafeInteger, Number);
+        }
       }
     }
 

ext/ffi/jit_trampoline.rs

@@ -58,12 +58,15 @@ fn native_to_c(ty: &NativeType) -> &'static str {
 
 pub(crate) fn codegen(sym: &crate::Symbol) -> String {
   let mut c = String::from(include_str!("prelude.h"));
-  let ret = native_to_c(&sym.result_type);
+  let needs_unwrap = crate::needs_unwrap(sym.result_type);
+
+  // Return type of the FFI call.
+  let ffi_ret = native_to_c(&sym.result_type);
+  // Return type of the trampoline.
+  let ret = if needs_unwrap { "void" } else { ffi_ret };
 
   // extern <return_type> func(
-  c += "\nextern ";
-  c += ret;
-  c += " func(";
+  let _ = write!(c, "\nextern {ffi_ret} func(");
   // <param_type> p0, <param_type> p1, ...);
   for (i, ty) in sym.parameter_types.iter().enumerate() {
     if i > 0 {
@@ -83,20 +86,35 @@ pub(crate) fn codegen(sym: &crate::Symbol) -> String {
     c += native_arg_to_c(ty);
     let _ = write!(c, " p{i}");
   }
-  c += ") {\n";
-  // return func(p0, p1, ...);
-  c += "  return func(";
-  for (i, ty) in sym.parameter_types.iter().enumerate() {
-    if i > 0 {
-      c += ", ";
-    }
-    if matches!(ty, NativeType::Pointer) {
-      let _ = write!(c, "p{i}->data");
-    } else {
-      let _ = write!(c, "p{i}");
-    }
+  if needs_unwrap {
+    let _ = write!(c, ", struct FastApiTypedArray* const p_ret");
   }
-  c += ");\n}\n\n";
+  c += ") {\n";
+  // func(p0, p1, ...);
+  let mut call_s = String::from("func(");
+  {
+    for (i, ty) in sym.parameter_types.iter().enumerate() {
+      if i > 0 {
+        call_s += ", ";
+      }
+      if matches!(ty, NativeType::Pointer) {
+        let _ = write!(call_s, "p{i}->data");
+      } else {
+        let _ = write!(call_s, "p{i}");
+      }
+    }
+    call_s += ");\n";
+  }
+  if needs_unwrap {
+    // <return_type> r = func(p0, p1, ...);
+    // ((<return_type>*)p_ret->data)[0] = r;
+    let _ = write!(c, " {ffi_ret} r = {call_s}");
+    let _ = writeln!(c, " (({ffi_ret}*)p_ret->data)[0] = r;");
+  } else {
+    // return func(p0, p1, ...);
+    let _ = write!(c, "  return {call_s}");
+  }
+  c += "}\n\n";
   c
 }
 
@@ -190,6 +208,22 @@ mod tests {
         \n  return func(p0->data, p1->data);\n\
       }\n\n",
     );
+    assert_codegen(
+      codegen(vec![], NativeType::U64),
+      "extern uint64_t func();\n\n\
+      void func_trampoline(void* recv, struct FastApiTypedArray* const p_ret) {\
+        \n uint64_t r = func();\
+        \n ((uint64_t*)p_ret->data)[0] = r;\n\
+      }\n\n",
+    );
+    assert_codegen(
+      codegen(vec![NativeType::Pointer, NativeType::Pointer], NativeType::U64),
+      "extern uint64_t func(void* p0, void* p1);\n\n\
+      void func_trampoline(void* recv, struct FastApiTypedArray* p0, struct FastApiTypedArray* p1, struct FastApiTypedArray* const p_ret) {\
+        \n uint64_t r = func(p0->data, p1->data);\
+        \n ((uint64_t*)p_ret->data)[0] = r;\n\
+      }\n\n",
+    );
   }
 
   #[test]

ext/ffi/lib.rs

@@ -752,9 +752,8 @@ impl From<&NativeType> for fast_api::Type {
   }
 }
 
-#[cfg(not(target_os = "windows"))]
-fn is_fast_api_rv(rv: NativeType) -> bool {
-  !matches!(
+fn needs_unwrap(rv: NativeType) -> bool {
+  matches!(
     rv,
     NativeType::Function
       | NativeType::Pointer
@@ -765,6 +764,10 @@ fn is_fast_api_rv(rv: NativeType) -> bool {
   )
 }
 
+fn is_i64(rv: NativeType) -> bool {
+  matches!(rv, NativeType::I64 | NativeType::ISize)
+}
+
 // Create a JavaScript function for synchronous FFI call to
 // the given symbol.
 fn make_sync_fn<'s>(
@@ -780,8 +783,12 @@ fn make_sync_fn<'s>(
   #[cfg(not(target_os = "windows"))]
   let mut fast_allocations: Option<*mut ()> = None;
   #[cfg(not(target_os = "windows"))]
-  if !sym.can_callback && is_fast_api_rv(sym.result_type) {
-    let ret = fast_api::Type::from(&sym.result_type);
+  if !sym.can_callback {
+    let needs_unwrap = needs_unwrap(sym.result_type);
+    let ret = match needs_unwrap {
+      true => fast_api::Type::Void,
+      false => fast_api::Type::from(&sym.result_type),
+    };
 
     let mut args = sym
       .parameter_types
@@ -790,6 +797,9 @@ fn make_sync_fn<'s>(
       .collect::<Vec<_>>();
     // recv
     args.insert(0, fast_api::Type::V8Value);
+    if needs_unwrap {
+      args.push(fast_api::Type::TypedArray(fast_api::CType::Int32));
+    }
     let symbol_trampoline =
       jit_trampoline::gen_trampoline(sym.clone()).expect("gen_trampoline");
     fast_ffi_templ = Some(FfiFastCallTemplate {
@@ -810,11 +820,46 @@ fn make_sync_fn<'s>(
       // SAFETY: The pointer will not be deallocated until the function is
       // garbage collected.
       let symbol = unsafe { &*(external.value() as *const Symbol) };
+      let needs_unwrap = match needs_unwrap(symbol.result_type) {
+        true => Some(args.get(symbol.parameter_types.len() as i32)),
+        false => None,
+      };
       match ffi_call_sync(scope, args, symbol) {
         Ok(result) => {
-          // SAFETY: Same return type declared to libffi; trust user to have it right beyond that.
-          let result = unsafe { result.to_v8(scope, symbol.result_type) };
-          rv.set(result.v8_value);
+          match needs_unwrap {
+            Some(v) => {
+              let view: v8::Local<v8::ArrayBufferView> = v.try_into().unwrap();
+              let backing_store =
+                view.buffer(scope).unwrap().get_backing_store();
+
+              if is_i64(symbol.result_type) {
+                // SAFETY: v8::SharedRef<v8::BackingStore> is similar to Arc<[u8]>,
+                // it points to a fixed continuous slice of bytes on the heap.
+                let bs = unsafe {
+                  &mut *(&backing_store[..] as *const _ as *mut [u8]
+                    as *mut i64)
+                };
+                // SAFETY: We already checked that type == I64
+                let value = unsafe { result.i64_value };
+                *bs = value;
+              } else {
+                // SAFETY: v8::SharedRef<v8::BackingStore> is similar to Arc<[u8]>,
+                // it points to a fixed continuous slice of bytes on the heap.
+                let bs = unsafe {
+                  &mut *(&backing_store[..] as *const _ as *mut [u8]
+                    as *mut u64)
+                };
+                // SAFETY: We checked that type == U64
+                let value = unsafe { result.u64_value };
+                *bs = value;
+              }
+            }
+            None => {
+              // SAFETY: Same return type declared to libffi; trust user to have it right beyond that.
+              let result = unsafe { result.to_v8(scope, symbol.result_type) };
+              rv.set(result.v8_value);
+            }
+          }
         }
         Err(err) => {
           deno_core::_ops::throw_type_error(scope, err.to_string());

test_ffi/src/lib.rs

@@ -404,3 +404,10 @@ pub struct Structure {
 
 #[no_mangle]
 pub static mut static_ptr: Structure = Structure { _data: 42 };
+
+static STRING: &str = "Hello, world!\0";
+
+#[no_mangle]
+extern "C" fn ffi_string() -> *const u8 {
+  STRING.as_ptr()
+}

test_ffi/tests/bench.js

@@ -12,6 +12,8 @@ const libPath = `${targetDir}/${libPrefix}test_ffi.${libSuffix}`;
 const dylib = Deno.dlopen(libPath, {
   "nop": { parameters: [], result: "void" },
   "add_u32": { parameters: ["u32", "u32"], result: "u32" },
+  "add_u64": { parameters: ["u64", "u64"], result: "u64" },
+  "ffi_string": { parameters: [], result: "pointer" },
   "hash": { parameters: ["pointer", "u32"], result: "u32" },
   "nop_u8": { parameters: ["u8"], result: "void" },
   "nop_i8": { parameters: ["i8"], result: "void" },
@@ -227,11 +229,6 @@ Deno.bench("nop()", () => {
   nop();
 });
 
-const { add_u32 } = dylib.symbols;
-Deno.bench("add_u32()", () => {
-  add_u32(1, 2);
-});
-
 const bytes = new Uint8Array(64);
 
 const { hash } = dylib.symbols;
@@ -239,6 +236,37 @@ Deno.bench("hash()", () => {
   hash(bytes, bytes.byteLength);
 });
 
+const { ffi_string } = dylib.symbols;
+Deno.bench(
+  "c string",
+  () => new Deno.UnsafePointerView(ffi_string()).getCString(),
+);
+
+const { add_u32 } = dylib.symbols;
+Deno.bench("add_u32()", () => {
+  add_u32(1, 2);
+});
+
+const { return_buffer } = dylib.symbols;
+Deno.bench("return_buffer()", () => {
+  return_buffer();
+});
+
+const { add_u64 } = dylib.symbols;
+Deno.bench("add_u64()", () => {
+  add_u64(1, 2);
+});
+
+const { return_u64 } = dylib.symbols;
+Deno.bench("return_u64()", () => {
+  return_u64();
+});
+
+const { return_i64 } = dylib.symbols;
+Deno.bench("return_i64()", () => {
+  return_i64();
+});
+
 const { nop_u8 } = dylib.symbols;
 Deno.bench("nop_u8()", () => {
   nop_u8(100);
@@ -348,16 +376,6 @@ Deno.bench("return_i32()", () => {
   return_i32();
 });
 
-const { return_u64 } = dylib.symbols;
-Deno.bench("return_u64()", () => {
-  return_u64();
-});
-
-const { return_i64 } = dylib.symbols;
-Deno.bench("return_i64()", () => {
-  return_i64();
-});
-
 const { return_usize } = dylib.symbols;
 Deno.bench("return_usize()", () => {
   return_usize();
@@ -378,11 +396,6 @@ Deno.bench("return_f64()", () => {
   return_f64();
 });
 
-const { return_buffer } = dylib.symbols;
-Deno.bench("return_buffer()", () => {
-  return_buffer();
-});
-
 // Nonblocking calls
 
 const { nop_nonblocking } = dylib.symbols;

test_ffi/tests/test.js

@@ -197,7 +197,20 @@ dylib.symbols.print_buffer(buffer, buffer.length);
 const subarray = buffer.subarray(3);
 dylib.symbols.print_buffer(subarray, subarray.length - 2);
 dylib.symbols.print_buffer2(buffer, buffer.length, buffer2, buffer2.length);
-const ptr0 = dylib.symbols.return_buffer();
+
+const { return_buffer } = symbols;
+function returnBuffer() { return return_buffer(); };
+
+%PrepareFunctionForOptimization(returnBuffer);
+returnBuffer();
+%OptimizeFunctionOnNextCall(returnBuffer);
+const ptr0 = returnBuffer();
+
+const status = %GetOptimizationStatus(returnBuffer);
+if (!(status & (1 << 4))) {
+  throw new Error("returnBuffer is not optimized");
+}
+
 dylib.symbols.print_buffer(ptr0, 8);
 const ptrView = new Deno.UnsafePointerView(ptr0);
 const into = new Uint8Array(6);