diff --git a/ext/ffi/00_ffi.js b/ext/ffi/00_ffi.js index bc03dfb144..4d77449e88 100644 --- a/ext/ffi/00_ffi.js +++ b/ext/ffi/00_ffi.js @@ -7,9 +7,17 @@ const { BigInt, ObjectDefineProperty, + ArrayPrototypeMap, + Number, + NumberIsSafeInteger, + ArrayPrototypeJoin, ObjectPrototypeIsPrototypeOf, PromisePrototypeThen, TypeError, + Int32Array, + Uint32Array, + BigInt64Array, + Function, } = window.__bootstrap.primordials; function unpackU64(returnValue) { @@ -209,6 +217,10 @@ type === "usize" || type === "isize"; } + function isI64(type) { + return type === "i64" || type === "isize"; + } + class UnsafeCallback { #refcount; #rid; @@ -293,11 +305,11 @@ ); continue; } + const resultType = symbols[symbol].result; + const needsUnpacking = isReturnedAsBigInt(resultType); const isNonBlocking = symbols[symbol].nonblocking; if (isNonBlocking) { - const resultType = symbols[symbol].result; - const needsUnpacking = isReturnedAsBigInt(resultType); ObjectDefineProperty( this.symbols, symbol, @@ -326,6 +338,38 @@ }, ); } + + if (needsUnpacking && !isNonBlocking) { + const call = this.symbols[symbol]; + const parameters = symbols[symbol].parameters; + const vi = new Int32Array(2); + const vui = new Uint32Array(vi.buffer); + const b = new BigInt64Array(vi.buffer); + + const params = ArrayPrototypeJoin( + ArrayPrototypeMap(parameters, (_, index) => `p${index}`), + ", ", + ); + // Make sure V8 has no excuse to not optimize this function. + this.symbols[symbol] = new Function( + "vi", + "vui", + "b", + "call", + "NumberIsSafeInteger", + "Number", + `return function (${params}) { + call(${params}${parameters.length > 0 ? ", " : ""}vi); + ${ + isI64(resultType) + ? `const n1 = Number(b[0])` + : `const n1 = vui[0] + 2 ** 32 * vui[1]` // Faster path for u64 + }; + if (NumberIsSafeInteger(n1)) return n1; + return b[0]; + }`, + )(vi, vui, b, call, NumberIsSafeInteger, Number); + } } } diff --git a/ext/ffi/jit_trampoline.rs b/ext/ffi/jit_trampoline.rs index 92e63348d2..4785fd0920 100644 --- a/ext/ffi/jit_trampoline.rs +++ b/ext/ffi/jit_trampoline.rs @@ -58,12 +58,15 @@ fn native_to_c(ty: &NativeType) -> &'static str { pub(crate) fn codegen(sym: &crate::Symbol) -> String { let mut c = String::from(include_str!("prelude.h")); - let ret = native_to_c(&sym.result_type); + let needs_unwrap = crate::needs_unwrap(sym.result_type); + + // Return type of the FFI call. + let ffi_ret = native_to_c(&sym.result_type); + // Return type of the trampoline. + let ret = if needs_unwrap { "void" } else { ffi_ret }; // extern func( - c += "\nextern "; - c += ret; - c += " func("; + let _ = write!(c, "\nextern {ffi_ret} func("); // p0, p1, ...); for (i, ty) in sym.parameter_types.iter().enumerate() { if i > 0 { @@ -83,20 +86,35 @@ pub(crate) fn codegen(sym: &crate::Symbol) -> String { c += native_arg_to_c(ty); let _ = write!(c, " p{i}"); } - c += ") {\n"; - // return func(p0, p1, ...); - c += " return func("; - for (i, ty) in sym.parameter_types.iter().enumerate() { - if i > 0 { - c += ", "; - } - if matches!(ty, NativeType::Pointer) { - let _ = write!(c, "p{i}->data"); - } else { - let _ = write!(c, "p{i}"); - } + if needs_unwrap { + let _ = write!(c, ", struct FastApiTypedArray* const p_ret"); } - c += ");\n}\n\n"; + c += ") {\n"; + // func(p0, p1, ...); + let mut call_s = String::from("func("); + { + for (i, ty) in sym.parameter_types.iter().enumerate() { + if i > 0 { + call_s += ", "; + } + if matches!(ty, NativeType::Pointer) { + let _ = write!(call_s, "p{i}->data"); + } else { + let _ = write!(call_s, "p{i}"); + } + } + call_s += ");\n"; + } + if needs_unwrap { + // r = func(p0, p1, ...); + // ((*)p_ret->data)[0] = r; + let _ = write!(c, " {ffi_ret} r = {call_s}"); + let _ = writeln!(c, " (({ffi_ret}*)p_ret->data)[0] = r;"); + } else { + // return func(p0, p1, ...); + let _ = write!(c, " return {call_s}"); + } + c += "}\n\n"; c } @@ -190,6 +208,22 @@ mod tests { \n return func(p0->data, p1->data);\n\ }\n\n", ); + assert_codegen( + codegen(vec![], NativeType::U64), + "extern uint64_t func();\n\n\ + void func_trampoline(void* recv, struct FastApiTypedArray* const p_ret) {\ + \n uint64_t r = func();\ + \n ((uint64_t*)p_ret->data)[0] = r;\n\ + }\n\n", + ); + assert_codegen( + codegen(vec![NativeType::Pointer, NativeType::Pointer], NativeType::U64), + "extern uint64_t func(void* p0, void* p1);\n\n\ + void func_trampoline(void* recv, struct FastApiTypedArray* p0, struct FastApiTypedArray* p1, struct FastApiTypedArray* const p_ret) {\ + \n uint64_t r = func(p0->data, p1->data);\ + \n ((uint64_t*)p_ret->data)[0] = r;\n\ + }\n\n", + ); } #[test] diff --git a/ext/ffi/lib.rs b/ext/ffi/lib.rs index 05015a45a7..ec80d47864 100644 --- a/ext/ffi/lib.rs +++ b/ext/ffi/lib.rs @@ -752,9 +752,8 @@ impl From<&NativeType> for fast_api::Type { } } -#[cfg(not(target_os = "windows"))] -fn is_fast_api_rv(rv: NativeType) -> bool { - !matches!( +fn needs_unwrap(rv: NativeType) -> bool { + matches!( rv, NativeType::Function | NativeType::Pointer @@ -765,6 +764,10 @@ fn is_fast_api_rv(rv: NativeType) -> bool { ) } +fn is_i64(rv: NativeType) -> bool { + matches!(rv, NativeType::I64 | NativeType::ISize) +} + // Create a JavaScript function for synchronous FFI call to // the given symbol. fn make_sync_fn<'s>( @@ -780,8 +783,12 @@ fn make_sync_fn<'s>( #[cfg(not(target_os = "windows"))] let mut fast_allocations: Option<*mut ()> = None; #[cfg(not(target_os = "windows"))] - if !sym.can_callback && is_fast_api_rv(sym.result_type) { - let ret = fast_api::Type::from(&sym.result_type); + if !sym.can_callback { + let needs_unwrap = needs_unwrap(sym.result_type); + let ret = match needs_unwrap { + true => fast_api::Type::Void, + false => fast_api::Type::from(&sym.result_type), + }; let mut args = sym .parameter_types @@ -790,6 +797,9 @@ fn make_sync_fn<'s>( .collect::>(); // recv args.insert(0, fast_api::Type::V8Value); + if needs_unwrap { + args.push(fast_api::Type::TypedArray(fast_api::CType::Int32)); + } let symbol_trampoline = jit_trampoline::gen_trampoline(sym.clone()).expect("gen_trampoline"); fast_ffi_templ = Some(FfiFastCallTemplate { @@ -810,11 +820,46 @@ fn make_sync_fn<'s>( // SAFETY: The pointer will not be deallocated until the function is // garbage collected. let symbol = unsafe { &*(external.value() as *const Symbol) }; + let needs_unwrap = match needs_unwrap(symbol.result_type) { + true => Some(args.get(symbol.parameter_types.len() as i32)), + false => None, + }; match ffi_call_sync(scope, args, symbol) { Ok(result) => { - // SAFETY: Same return type declared to libffi; trust user to have it right beyond that. - let result = unsafe { result.to_v8(scope, symbol.result_type) }; - rv.set(result.v8_value); + match needs_unwrap { + Some(v) => { + let view: v8::Local = v.try_into().unwrap(); + let backing_store = + view.buffer(scope).unwrap().get_backing_store(); + + if is_i64(symbol.result_type) { + // SAFETY: v8::SharedRef is similar to Arc<[u8]>, + // it points to a fixed continuous slice of bytes on the heap. + let bs = unsafe { + &mut *(&backing_store[..] as *const _ as *mut [u8] + as *mut i64) + }; + // SAFETY: We already checked that type == I64 + let value = unsafe { result.i64_value }; + *bs = value; + } else { + // SAFETY: v8::SharedRef is similar to Arc<[u8]>, + // it points to a fixed continuous slice of bytes on the heap. + let bs = unsafe { + &mut *(&backing_store[..] as *const _ as *mut [u8] + as *mut u64) + }; + // SAFETY: We checked that type == U64 + let value = unsafe { result.u64_value }; + *bs = value; + } + } + None => { + // SAFETY: Same return type declared to libffi; trust user to have it right beyond that. + let result = unsafe { result.to_v8(scope, symbol.result_type) }; + rv.set(result.v8_value); + } + } } Err(err) => { deno_core::_ops::throw_type_error(scope, err.to_string()); diff --git a/test_ffi/src/lib.rs b/test_ffi/src/lib.rs index bc89bd66c6..b061ca8d5b 100644 --- a/test_ffi/src/lib.rs +++ b/test_ffi/src/lib.rs @@ -404,3 +404,10 @@ pub struct Structure { #[no_mangle] pub static mut static_ptr: Structure = Structure { _data: 42 }; + +static STRING: &str = "Hello, world!\0"; + +#[no_mangle] +extern "C" fn ffi_string() -> *const u8 { + STRING.as_ptr() +} diff --git a/test_ffi/tests/bench.js b/test_ffi/tests/bench.js index da29f482fa..0846155756 100644 --- a/test_ffi/tests/bench.js +++ b/test_ffi/tests/bench.js @@ -12,6 +12,8 @@ const libPath = `${targetDir}/${libPrefix}test_ffi.${libSuffix}`; const dylib = Deno.dlopen(libPath, { "nop": { parameters: [], result: "void" }, "add_u32": { parameters: ["u32", "u32"], result: "u32" }, + "add_u64": { parameters: ["u64", "u64"], result: "u64" }, + "ffi_string": { parameters: [], result: "pointer" }, "hash": { parameters: ["pointer", "u32"], result: "u32" }, "nop_u8": { parameters: ["u8"], result: "void" }, "nop_i8": { parameters: ["i8"], result: "void" }, @@ -227,11 +229,6 @@ Deno.bench("nop()", () => { nop(); }); -const { add_u32 } = dylib.symbols; -Deno.bench("add_u32()", () => { - add_u32(1, 2); -}); - const bytes = new Uint8Array(64); const { hash } = dylib.symbols; @@ -239,6 +236,37 @@ Deno.bench("hash()", () => { hash(bytes, bytes.byteLength); }); +const { ffi_string } = dylib.symbols; +Deno.bench( + "c string", + () => new Deno.UnsafePointerView(ffi_string()).getCString(), +); + +const { add_u32 } = dylib.symbols; +Deno.bench("add_u32()", () => { + add_u32(1, 2); +}); + +const { return_buffer } = dylib.symbols; +Deno.bench("return_buffer()", () => { + return_buffer(); +}); + +const { add_u64 } = dylib.symbols; +Deno.bench("add_u64()", () => { + add_u64(1, 2); +}); + +const { return_u64 } = dylib.symbols; +Deno.bench("return_u64()", () => { + return_u64(); +}); + +const { return_i64 } = dylib.symbols; +Deno.bench("return_i64()", () => { + return_i64(); +}); + const { nop_u8 } = dylib.symbols; Deno.bench("nop_u8()", () => { nop_u8(100); @@ -348,16 +376,6 @@ Deno.bench("return_i32()", () => { return_i32(); }); -const { return_u64 } = dylib.symbols; -Deno.bench("return_u64()", () => { - return_u64(); -}); - -const { return_i64 } = dylib.symbols; -Deno.bench("return_i64()", () => { - return_i64(); -}); - const { return_usize } = dylib.symbols; Deno.bench("return_usize()", () => { return_usize(); @@ -378,11 +396,6 @@ Deno.bench("return_f64()", () => { return_f64(); }); -const { return_buffer } = dylib.symbols; -Deno.bench("return_buffer()", () => { - return_buffer(); -}); - // Nonblocking calls const { nop_nonblocking } = dylib.symbols; diff --git a/test_ffi/tests/test.js b/test_ffi/tests/test.js index e27a09d4f0..d658ec1697 100644 --- a/test_ffi/tests/test.js +++ b/test_ffi/tests/test.js @@ -197,7 +197,20 @@ dylib.symbols.print_buffer(buffer, buffer.length); const subarray = buffer.subarray(3); dylib.symbols.print_buffer(subarray, subarray.length - 2); dylib.symbols.print_buffer2(buffer, buffer.length, buffer2, buffer2.length); -const ptr0 = dylib.symbols.return_buffer(); + +const { return_buffer } = symbols; +function returnBuffer() { return return_buffer(); }; + +%PrepareFunctionForOptimization(returnBuffer); +returnBuffer(); +%OptimizeFunctionOnNextCall(returnBuffer); +const ptr0 = returnBuffer(); + +const status = %GetOptimizationStatus(returnBuffer); +if (!(status & (1 << 4))) { + throw new Error("returnBuffer is not optimized"); +} + dylib.symbols.print_buffer(ptr0, 8); const ptrView = new Deno.UnsafePointerView(ptr0); const into = new Uint8Array(6);