feat: Deno.core.{encode,decode}; standalone UTF-8 encoding/decoding (#4349)

This commits add two new methods to "Deno.core" namespace: "encode" and "decode".

Those methods are bound in Rust to provide a) fast b) generally available of encoding and decoding UTF-8 strings. 

Both methods are now used in "cli/js/dispatch_json.ts".

cli/js/globals.ts

@@ -97,6 +97,9 @@ declare global {
     evalContext(code: string): [any, EvalErrorInfo | null];
 
     formatError: (e: Error) => string;
+
+    decode(bytes: Uint8Array): string;
+    encode(text: string): Uint8Array;
   }
 
   // Only `var` variables show up in the `globalThis` type when doing a global

cli/js/ops/dispatch_json.ts

@@ -1,6 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 import * as util from "../util.ts";
-import { TextEncoder, TextDecoder } from "../web/text_encoding.ts";
 import { core } from "../core.ts";
 import { OPS_CACHE } from "../runtime.ts";
 import { ErrorKind, getErrorClass } from "../errors.ts";
@@ -30,13 +29,13 @@ function nextPromiseId(): number {
 }
 
 function decode(ui8: Uint8Array): JsonResponse {
-  const s = new TextDecoder().decode(ui8);
+  const s = core.decode(ui8);
   return JSON.parse(s) as JsonResponse;
 }
 
 function encode(args: object): Uint8Array {
   const s = JSON.stringify(args);
-  return new TextEncoder().encode(s);
+  return core.encode(s);
 }
 
 function unwrapResponse(res: JsonResponse): Ok {

cli/tests/core_decode_perf.js

@@ -0,0 +1,37 @@
+const mixed = new TextEncoder().encode("@Ā๐😀");
+
+function generateRandom(bytes) {
+  const result = new Uint8Array(bytes);
+  let i = 0;
+  while (i < bytes) {
+    const toAdd = Math.floor(Math.random() * Math.min(4, bytes - i));
+    switch (toAdd) {
+      case 0:
+        result[i] = mixed[0];
+        i++;
+        break;
+      case 1:
+        result[i] = mixed[1];
+        result[i + 1] = mixed[2];
+        i += 2;
+        break;
+      case 2:
+        result[i] = mixed[3];
+        result[i + 1] = mixed[4];
+        result[i + 2] = mixed[5];
+        i += 3;
+        break;
+      case 3:
+        result[i] = mixed[6];
+        result[i + 1] = mixed[7];
+        result[i + 2] = mixed[8];
+        result[i + 3] = mixed[9];
+        i += 4;
+        break;
+    }
+  }
+  return result;
+}
+
+const randomData = generateRandom(1024);
+for (let i = 0; i < 10_000; i++) Deno.core.decode(randomData);

cli/tests/core_encode_perf.js

@@ -0,0 +1,32 @@
+const mixed = "@Ā๐😀";
+
+function generateRandom(bytes) {
+  let result = "";
+  let i = 0;
+  while (i < bytes) {
+    const toAdd = Math.floor(Math.random() * Math.min(4, bytes - i));
+    switch (toAdd) {
+      case 0:
+        result += mixed[0];
+        i++;
+        break;
+      case 1:
+        result += mixed[1];
+        i++;
+        break;
+      case 2:
+        result += mixed[2];
+        i++;
+        break;
+      case 3:
+        result += mixed[3];
+        result += mixed[4];
+        i += 2;
+        break;
+    }
+  }
+  return result;
+}
+
+const randomData = generateRandom(1024);
+for (let i = 0; i < 10_000; i++) Deno.core.encode(randomData);

core/bindings.rs

@@ -35,6 +35,12 @@ lazy_static! {
       v8::ExternalReference {
         function: queue_microtask.map_fn_to()
       },
+      v8::ExternalReference {
+        function: encode.map_fn_to()
+      },
+      v8::ExternalReference {
+        function: decode.map_fn_to()
+      },
     ]);
 }
 
@@ -156,6 +162,22 @@ pub fn initialize_context<'s>(
     format_error_val.into(),
   );
 
+  let mut encode_tmpl = v8::FunctionTemplate::new(scope, encode);
+  let encode_val = encode_tmpl.get_function(scope, context).unwrap();
+  core_val.set(
+    context,
+    v8::String::new(scope, "encode").unwrap().into(),
+    encode_val.into(),
+  );
+
+  let mut decode_tmpl = v8::FunctionTemplate::new(scope, decode);
+  let decode_val = decode_tmpl.get_function(scope, context).unwrap();
+  core_val.set(
+    context,
+    v8::String::new(scope, "decode").unwrap().into(),
+    decode_val.into(),
+  );
+
   core_val.set_accessor(
     context,
     v8::String::new(scope, "shared").unwrap().into(),
@@ -551,6 +573,52 @@ fn format_error(
   rv.set(e.into())
 }
 
+fn encode(
+  scope: v8::FunctionCallbackScope,
+  args: v8::FunctionCallbackArguments,
+  mut rv: v8::ReturnValue,
+) {
+  let text = match v8::Local::<v8::String>::try_from(args.get(0)) {
+    Ok(s) => s,
+    Err(_) => {
+      let msg = v8::String::new(scope, "Invalid argument").unwrap();
+      let exception = v8::Exception::type_error(scope, msg);
+      scope.isolate().throw_exception(exception);
+      return;
+    }
+  };
+  let text_str = text.to_rust_string_lossy(scope);
+  let text_bytes = text_str.as_bytes().to_vec().into_boxed_slice();
+  let buf = boxed_slice_to_uint8array(scope, text_bytes);
+  rv.set(buf.into())
+}
+
+fn decode(
+  scope: v8::FunctionCallbackScope,
+  args: v8::FunctionCallbackArguments,
+  mut rv: v8::ReturnValue,
+) {
+  let buf = match v8::Local::<v8::ArrayBufferView>::try_from(args.get(0)) {
+    Ok(view) => {
+      let byte_offset = view.byte_offset();
+      let byte_length = view.byte_length();
+      let backing_store = view.buffer().unwrap().get_backing_store();
+      let buf = unsafe { &**backing_store.get() };
+      &buf[byte_offset..byte_offset + byte_length]
+    }
+    Err(..) => {
+      let msg = v8::String::new(scope, "Invalid argument").unwrap();
+      let exception = v8::Exception::type_error(scope, msg);
+      scope.isolate().throw_exception(exception);
+      return;
+    }
+  };
+
+  let text_str =
+    v8::String::new_from_utf8(scope, &buf, v8::NewStringType::Normal).unwrap();
+  rv.set(text_str.into())
+}
+
 fn queue_microtask(
   scope: v8::FunctionCallbackScope,
   args: v8::FunctionCallbackArguments,

core/encode_decode_test.js

@@ -0,0 +1,40 @@
+// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+
+function assertArrayEquals(a1, a2) {
+  if (a1.length !== a2.length) throw Error("assert");
+
+  for (const index in a1) {
+    if (a1[index] !== a2[index]) {
+      throw Error("assert");
+    }
+  }
+}
+
+function main() {
+  // prettier-ignore
+  const fixture1 = [
+    0xf0, 0x9d, 0x93, 0xbd,
+    0xf0, 0x9d, 0x93, 0xae,
+    0xf0, 0x9d, 0x94, 0x81,
+    0xf0, 0x9d, 0x93, 0xbd
+  ];
+  // prettier-ignore
+  const fixture2 = [ 
+    72, 101, 108, 108, 
+    111, 32, 239, 191, 
+    189, 239, 191, 189, 
+    32, 87, 111, 114, 
+    108, 100 
+  ];
+
+  assertArrayEquals(Array.from(Deno.core.encode("𝓽𝓮𝔁𝓽")), fixture1);
+  assertArrayEquals(
+    Array.from(Deno.core.encode("Hello \udc12\ud834 World")),
+    fixture2
+  );
+
+  assert(Deno.core.decode(new Uint8Array(fixture1)) === "𝓽𝓮𝔁𝓽");
+  assert(Deno.core.decode(new Uint8Array(fixture2)) === "Hello <20><> World");
+}
+
+main();

core/isolate.rs

@@ -1112,6 +1112,20 @@ pub mod tests {
     });
   }
 
+  #[test]
+  fn test_encode_decode() {
+    run_in_task(|mut cx| {
+      let (mut isolate, _dispatch_count) = setup(Mode::Async);
+      js_check(isolate.execute(
+        "encode_decode_test.js",
+        include_str!("encode_decode_test.js"),
+      ));
+      if let Poll::Ready(Err(_)) = isolate.poll_unpin(&mut cx) {
+        unreachable!();
+      }
+    });
+  }
+
   #[test]
   fn will_snapshot() {
     let snapshot = {

tools/benchmark.py

@@ -28,7 +28,9 @@ exec_time_benchmarks = [
     ("workers_startup", ["cli/tests/workers_startup_bench.ts"]),
     ("workers_round_robin", ["cli/tests/workers_round_robin_bench.ts"]),
     ("text_decoder", ["cli/tests/text_decoder_perf.js"]),
+    ("core_decode", ["cli/tests/core_decode_perf.js"]),
     ("text_encoder", ["cli/tests/text_encoder_perf.js"]),
+    ("core_encode", ["cli/tests/core_encode_perf.js"]),
 ]