Loader: support .wasm imports (#3328)

* loader: support .wasm imports

* http_server: true

* Support named exports

* Clippy

.eslintignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 std/deno.d.ts
 std/prettier/vendor

.prettierignore

@@ -1,3 +1,4 @@
+cli/compilers/wasm_wrap.js
 cli/tests/error_syntax.js
 cli/tests/badly_formatted.js
 cli/tests/top_level_for_await.js

Cargo.lock

@@ -91,6 +91,11 @@ dependencies = [
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "base64"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "bitflags"
 version = "1.2.1"
@@ -282,6 +287,7 @@ version = "0.23.0"
 dependencies = [
  "ansi_term 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "deno 0.23.0",
@@ -1944,6 +1950,7 @@ dependencies = [
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
 "checksum backtrace-sys 0.1.32 (registry+https://github.com/rust-lang/crates.io-index)" = "5d6575f128516de27e3ce99689419835fce9643a9b215a14d2b5b685be018491"
 "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
 "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 "checksum blake2b_simd 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "b83b7baab1e671718d78204225800d6b170e648188ac7dc992e9d6bddf87d0c0"
 "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708"

cli/Cargo.toml

@@ -27,6 +27,7 @@ deno_typescript = { path = "../deno_typescript", version = "0.23.0" }
 
 ansi_term = "0.12.1"
 atty = "0.2.13"
+base64 = "0.11.0"
 byteorder = "1.3.2"
 clap = "2.33.0"
 dirs = "2.0.2"

cli/compilers/mod.rs

@@ -5,10 +5,12 @@ use futures::Future;
 mod js;
 mod json;
 mod ts;
+mod wasm;
 
 pub use js::JsCompiler;
 pub use json::JsonCompiler;
 pub use ts::TsCompiler;
+pub use wasm::WasmCompiler;
 
 #[derive(Debug, Clone)]
 pub struct CompiledModule {

cli/compilers/wasm.rs

@@ -0,0 +1,174 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use crate::compilers::CompiledModule;
+use crate::compilers::CompiledModuleFuture;
+use crate::file_fetcher::SourceFile;
+use crate::global_state::ThreadSafeGlobalState;
+use crate::startup_data;
+use crate::state::*;
+use crate::worker::Worker;
+use deno::Buf;
+use futures::Future;
+use futures::IntoFuture;
+use serde_derive::Deserialize;
+use serde_json;
+use std::collections::HashMap;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, Mutex};
+use url::Url;
+
+// TODO(kevinkassimo): This is a hack to encode/decode data as base64 string.
+// (Since Deno namespace might not be available, Deno.read can fail).
+// Binary data is already available through source_file.source_code.
+// If this is proven too wasteful in practice, refactor this.
+
+// Ref: https://webassembly.github.io/esm-integration/js-api/index.html#esm-integration
+// https://github.com/nodejs/node/blob/35ec01097b2a397ad0a22aac536fe07514876e21/lib/internal/modules/esm/translators.js#L190-L210
+
+// Dynamically construct JS wrapper with custom static imports and named exports.
+// Boots up an internal worker to resolve imports/exports through query from V8.
+
+static WASM_WRAP: &str = include_str!("./wasm_wrap.js");
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+struct WasmModuleInfo {
+  import_list: Vec<String>,
+  export_list: Vec<String>,
+}
+
+#[derive(Default)]
+pub struct WasmCompiler {
+  cache: Arc<Mutex<HashMap<Url, CompiledModule>>>,
+}
+
+impl WasmCompiler {
+  /// Create a new V8 worker with snapshot of WASM compiler and setup compiler's runtime.
+  fn setup_worker(global_state: ThreadSafeGlobalState) -> Worker {
+    let (int, ext) = ThreadSafeState::create_channels();
+    let worker_state =
+      ThreadSafeState::new(global_state.clone(), None, true, int)
+        .expect("Unable to create worker state");
+
+    // Count how many times we start the compiler worker.
+    global_state
+      .metrics
+      .compiler_starts
+      .fetch_add(1, Ordering::SeqCst);
+
+    let mut worker = Worker::new(
+      "WASM".to_string(),
+      startup_data::compiler_isolate_init(),
+      worker_state,
+      ext,
+    );
+    worker.execute("denoMain('WASM')").unwrap();
+    worker.execute("workerMain()").unwrap();
+    worker.execute("wasmCompilerMain()").unwrap();
+    worker
+  }
+
+  pub fn compile_async(
+    self: &Self,
+    global_state: ThreadSafeGlobalState,
+    source_file: &SourceFile,
+  ) -> Box<CompiledModuleFuture> {
+    let cache = self.cache.clone();
+    let maybe_cached = { cache.lock().unwrap().get(&source_file.url).cloned() };
+    if let Some(m) = maybe_cached {
+      return Box::new(futures::future::ok(m.clone()));
+    }
+    let cache_ = self.cache.clone();
+
+    debug!(">>>>> wasm_compile_async START");
+    let base64_data = base64::encode(&source_file.source_code);
+    let worker = WasmCompiler::setup_worker(global_state.clone());
+    let worker_ = worker.clone();
+    let url = source_file.url.clone();
+
+    let fut = worker
+      .post_message(
+        serde_json::to_string(&base64_data)
+          .unwrap()
+          .into_boxed_str()
+          .into_boxed_bytes(),
+      )
+      .into_future()
+      .then(move |_| worker)
+      .then(move |result| {
+        if let Err(err) = result {
+          // TODO(ry) Need to forward the error instead of exiting.
+          eprintln!("{}", err.to_string());
+          std::process::exit(1);
+        }
+        debug!("Sent message to worker");
+        worker_.get_message()
+      })
+      .map_err(|_| panic!("not handled"))
+      .and_then(move |maybe_msg: Option<Buf>| {
+        debug!("Received message from worker");
+        let json_msg = maybe_msg.unwrap();
+        let module_info: WasmModuleInfo =
+          serde_json::from_slice(&json_msg).unwrap();
+        debug!("WASM module info: {:#?}", &module_info);
+        let code = wrap_wasm_code(
+          &base64_data,
+          &module_info.import_list,
+          &module_info.export_list,
+        );
+        debug!("Generated code: {}", &code);
+        let module = CompiledModule {
+          code,
+          name: url.to_string(),
+        };
+        {
+          cache_.lock().unwrap().insert(url.clone(), module.clone());
+        }
+        debug!("<<<<< wasm_compile_async END");
+        Ok(module)
+      });
+    Box::new(fut)
+  }
+}
+
+fn build_single_import(index: usize, origin: &str) -> String {
+  let origin_json = serde_json::to_string(origin).unwrap();
+  format!(
+    r#"import * as m{} from {};
+importObject[{}] = m{};
+"#,
+    index, &origin_json, &origin_json, index
+  )
+}
+
+fn build_imports(imports: &[String]) -> String {
+  let mut code = String::from("");
+  for (index, origin) in imports.iter().enumerate() {
+    code.push_str(&build_single_import(index, origin));
+  }
+  code
+}
+
+fn build_single_export(name: &str) -> String {
+  format!("export const {} = instance.exports.{};\n", name, name)
+}
+
+fn build_exports(exports: &[String]) -> String {
+  let mut code = String::from("");
+  for e in exports {
+    code.push_str(&build_single_export(e));
+  }
+  code
+}
+
+fn wrap_wasm_code(
+  base64_data: &str,
+  imports: &[String],
+  exports: &[String],
+) -> String {
+  let imports_code = build_imports(imports);
+  let exports_code = build_exports(exports);
+  String::from(WASM_WRAP)
+    .replace("//IMPORTS\n", &imports_code)
+    .replace("//EXPORTS\n", &exports_code)
+    .replace("BASE64_DATA", base64_data)
+}

cli/compilers/wasm_wrap.js

@@ -0,0 +1,19 @@
+const importObject = Object.create(null);
+//IMPORTS
+
+function base64ToUint8Array(data) {
+  const binString = window.atob(data);
+  const size = binString.length;
+  const bytes = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+const buffer = base64ToUint8Array("BASE64_DATA");
+const compiled = await WebAssembly.compile(buffer);
+
+const instance = new WebAssembly.Instance(compiled, importObject);
+
+//EXPORTS

cli/file_fetcher.rs

@@ -491,6 +491,7 @@ fn map_file_extension(path: &Path) -> msg::MediaType {
       Some("jsx") => msg::MediaType::JSX,
       Some("mjs") => msg::MediaType::JavaScript,
       Some("json") => msg::MediaType::Json,
+      Some("wasm") => msg::MediaType::Wasm,
       _ => msg::MediaType::Unknown,
     },
   }
@@ -1503,6 +1504,10 @@ mod tests {
       map_file_extension(Path::new("foo/bar.json")),
       msg::MediaType::Json
     );
+    assert_eq!(
+      map_file_extension(Path::new("foo/bar.wasm")),
+      msg::MediaType::Wasm
+    );
     assert_eq!(
       map_file_extension(Path::new("foo/bar.txt")),
       msg::MediaType::Unknown
@@ -1544,6 +1549,10 @@ mod tests {
       map_content_type(Path::new("foo/bar.json"), None),
       msg::MediaType::Json
     );
+    assert_eq!(
+      map_content_type(Path::new("foo/bar.wasm"), None),
+      msg::MediaType::Wasm
+    );
     assert_eq!(
       map_content_type(Path::new("foo/bar"), None),
       msg::MediaType::Unknown

cli/global_state.rs

@@ -3,6 +3,7 @@ use crate::compilers::CompiledModule;
 use crate::compilers::JsCompiler;
 use crate::compilers::JsonCompiler;
 use crate::compilers::TsCompiler;
+use crate::compilers::WasmCompiler;
 use crate::deno_dir;
 use crate::deno_error::permission_denied;
 use crate::file_fetcher::SourceFileFetcher;
@@ -45,6 +46,7 @@ pub struct GlobalState {
   pub js_compiler: JsCompiler,
   pub json_compiler: JsonCompiler,
   pub ts_compiler: TsCompiler,
+  pub wasm_compiler: WasmCompiler,
   pub lockfile: Option<Mutex<Lockfile>>,
 }
 
@@ -111,6 +113,7 @@ impl ThreadSafeGlobalState {
       ts_compiler,
       js_compiler: JsCompiler {},
       json_compiler: JsonCompiler {},
+      wasm_compiler: WasmCompiler::default(),
       lockfile,
     };
 
@@ -130,6 +133,9 @@ impl ThreadSafeGlobalState {
       .and_then(move |out| match out.media_type {
         msg::MediaType::Unknown => state1.js_compiler.compile_async(&out),
         msg::MediaType::Json => state1.json_compiler.compile_async(&out),
+        msg::MediaType::Wasm => {
+          state1.wasm_compiler.compile_async(state1.clone(), &out)
+        }
         msg::MediaType::TypeScript
         | msg::MediaType::TSX
         | msg::MediaType::JSX => {

cli/js/compiler.ts

@@ -28,7 +28,8 @@ enum MediaType {
   TypeScript = 2,
   TSX = 3,
   Json = 4,
-  Unknown = 5
+  Wasm = 5,
+  Unknown = 6
 }
 
 // Warning! The values in this enum are duplicated in cli/msg.rs
@@ -44,8 +45,8 @@ enum CompilerRequestType {
 const console = new Console(core.print);
 window.console = console;
 window.workerMain = workerMain;
-function denoMain(): void {
+function denoMain(compilerType?: string): void {
-  os.start(true, "TS");
+  os.start(true, compilerType || "TS");
 }
 window["denoMain"] = denoMain;
 
@@ -371,6 +372,9 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
       return ts.Extension.Tsx;
     case MediaType.Json:
       return ts.Extension.Json;
+    case MediaType.Wasm:
+      // Custom marker for Wasm type.
+      return ts.Extension.Js;
     case MediaType.Unknown:
     default:
       throw TypeError("Cannot resolve extension.");
@@ -724,3 +728,47 @@ window.compilerMain = function compilerMain(): void {
     workerClose();
   };
 };
+
+function base64ToUint8Array(data: string): Uint8Array {
+  const binString = window.atob(data);
+  const size = binString.length;
+  const bytes = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    bytes[i] = binString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+window.wasmCompilerMain = function wasmCompilerMain(): void {
+  // workerMain should have already been called since a compiler is a worker.
+  window.onmessage = async ({
+    data: binary
+  }: {
+    data: string;
+  }): Promise<void> => {
+    const buffer = base64ToUint8Array(binary);
+    // @ts-ignore
+    const compiled = await WebAssembly.compile(buffer);
+
+    util.log(">>> WASM compile start");
+
+    const importList = Array.from(
+      // @ts-ignore
+      new Set(WebAssembly.Module.imports(compiled).map(({ module }) => module))
+    );
+    const exportList = Array.from(
+      // @ts-ignore
+      new Set(WebAssembly.Module.exports(compiled).map(({ name }) => name))
+    );
+
+    postMessage({
+      importList,
+      exportList
+    });
+
+    util.log("<<< WASM compile end");
+
+    // The compiler isolate exits after a single message.
+    workerClose();
+  };
+};

cli/msg.rs

@@ -74,7 +74,8 @@ pub enum MediaType {
   TypeScript = 2,
   TSX = 3,
   Json = 4,
-  Unknown = 5,
+  Wasm = 5,
+  Unknown = 6,
 }
 
 pub fn enum_name_media_type(mt: MediaType) -> &'static str {
@@ -84,6 +85,7 @@ pub fn enum_name_media_type(mt: MediaType) -> &'static str {
     MediaType::TypeScript => "TypeScript",
     MediaType::TSX => "TSX",
     MediaType::Json => "Json",
+    MediaType::Wasm => "Wasm",
     MediaType::Unknown => "Unknown",
   }
 }

cli/ops/compiler.rs

@@ -2,6 +2,7 @@
 use super::dispatch_json::{Deserialize, JsonOp, Value};
 use crate::futures::future::join_all;
 use crate::futures::Future;
+use crate::msg;
 use crate::ops::json_op;
 use crate::state::ThreadSafeState;
 use deno::Loader;
@@ -74,17 +75,44 @@ fn op_fetch_source_files(
     futures.push(fut);
   }
 
+  let global_state = state.global_state.clone();
+
   let future = join_all(futures)
     .map_err(ErrBox::from)
     .and_then(move |files| {
-      let res = files
+      // We want to get an array of futures that resolves to
+      let v: Vec<_> = files
         .into_iter()
         .map(|file| {
+          // Special handling of Wasm files:
+          // compile them into JS first!
+          // This allows TS to do correct export types.
+          if file.media_type == msg::MediaType::Wasm {
+            return futures::future::Either::A(
+              global_state
+                .wasm_compiler
+                .compile_async(global_state.clone(), &file)
+                .and_then(|compiled_mod| Ok((file, Some(compiled_mod.code)))),
+            );
+          }
+          futures::future::Either::B(futures::future::ok((file, None)))
+        })
+        .collect();
+      join_all(v)
+    })
+    .and_then(move |files_with_code| {
+      let res = files_with_code
+        .into_iter()
+        .map(|(file, maybe_code)| {
           json!({
             "url": file.url.to_string(),
             "filename": file.filename.to_str().unwrap(),
             "mediaType": file.media_type as i32,
-            "sourceCode": String::from_utf8(file.source_code).unwrap(),
+            "sourceCode": if let Some(code) = maybe_code {
+              code
+            } else {
+              String::from_utf8(file.source_code).unwrap()
+            },
           })
         })
         .collect();

cli/tests/051_wasm_import.ts

@@ -0,0 +1,22 @@
+import { add, addImported, addRemote } from "./051_wasm_import/simple.wasm";
+import { state } from "./051_wasm_import/wasm-dep.js";
+
+function assertEquals(actual: unknown, expected: unknown, msg?: string): void {
+  if (actual !== expected) {
+    throw new Error(msg);
+  }
+}
+
+assertEquals(state, "WASM Start Executed", "Incorrect state");
+
+assertEquals(add(10, 20), 30, "Incorrect add");
+
+assertEquals(addImported(0), 42, "Incorrect addImported");
+
+assertEquals(state, "WASM JS Function Executed", "Incorrect state");
+
+assertEquals(addImported(1), 43, "Incorrect addImported");
+
+assertEquals(addRemote(1), 2020, "Incorrect addRemote");
+
+console.log("Passed");

cli/tests/051_wasm_import.ts.out

@@ -0,0 +1 @@
+Passed

cli/tests/051_wasm_import/remote.ts

@@ -0,0 +1,3 @@
+export function jsRemoteFn(): number {
+  return 2019;
+}

cli/tests/051_wasm_import/simple.wat

@@ -0,0 +1,31 @@
+;; From https://github.com/nodejs/node/blob/bbc254db5db672643aad89a436a4938412a5704e/test/fixtures/es-modules/simple.wat
+;; MIT Licensed
+;; $ wat2wasm simple.wat -o simple.wasm
+
+(module
+  (import "./wasm-dep.js" "jsFn" (func $jsFn (result i32)))
+  (import "./wasm-dep.js" "jsInitFn" (func $jsInitFn))
+  (import "http://127.0.0.1:4545/cli/tests/051_wasm_import/remote.ts" "jsRemoteFn" (func $jsRemoteFn (result i32)))
+  (export "add" (func $add))
+  (export "addImported" (func $addImported))
+  (export "addRemote" (func $addRemote))
+  (start $startFn)
+  (func $startFn
+    call $jsInitFn
+  )
+  (func $add (param $a i32) (param $b i32) (result i32)
+    local.get $a
+    local.get $b
+    i32.add
+  )
+  (func $addImported (param $a i32) (result i32)
+    local.get $a
+    call $jsFn
+    i32.add
+  )
+  (func $addRemote (param $a i32) (result i32)
+    local.get $a
+    call $jsRemoteFn
+    i32.add
+  )
+)

cli/tests/051_wasm_import/wasm-dep.js

@@ -0,0 +1,17 @@
+function assertEquals(actual, expected, msg) {
+  if (actual !== expected) {
+    throw new Error(msg || "");
+  }
+}
+
+export function jsFn() {
+  state = "WASM JS Function Executed";
+  return 42;
+}
+
+export let state = "JS Function Executed";
+
+export function jsInitFn() {
+  assertEquals(state, "JS Function Executed", "Incorrect state");
+  state = "WASM Start Executed";
+}

cli/tests/integration_tests.rs

@@ -356,6 +356,12 @@ itest!(_050_more_jsons {
   output: "050_more_jsons.ts.out",
 });
 
+itest!(_051_wasm_import {
+  args: "run --reload --allow-net --allow-read 051_wasm_import.ts",
+  output: "051_wasm_import.ts.out",
+  http_server: true,
+});
+
 itest!(lock_check_ok {
   args: "run --lock=lock_check_ok.json http://127.0.0.1:4545/cli/tests/003_relative_import.ts",
   output: "003_relative_import.ts.out",

tools/lint.py

@@ -39,9 +39,10 @@ def eslint():
     script = os.path.join(third_party_path, "node_modules", "eslint", "bin",
                           "eslint")
     # Find all *directories* in the main repo that contain .ts/.js files.
-    source_files = git_ls_files(
+    source_files = git_ls_files(root_path, [
-        root_path,
+        "*.js", "*.ts", ":!:std/prettier/vendor/*", ":!:std/**/testdata/*",
-        ["*.js", "*.ts", ":!:std/prettier/vendor/*", ":!:std/**/testdata/*"])
+        ":!:cli/compilers/*"
+    ])
     source_dirs = set([os.path.dirname(f) for f in source_files])
     # Within the source dirs, eslint does its own globbing, taking into account
     # the exclusion rules listed in '.eslintignore'.