Add WebAssembly to runtime library (#1677)

This also modifies the `ts_library_builder` to support inlining assets.

Includes integration tests from @sh7dm

BUILD.gn

@@ -76,6 +76,7 @@ ts_sources = [
   "js/headers.ts",
   "js/io.ts",
   "js/libdeno.ts",
+  "js/lib.web_assembly.d.ts",
   "js/main.ts",
   "js/make_temp_dir.ts",
   "js/metrics.ts",
@@ -186,6 +187,8 @@ run_node("deno_runtime_declaration") {
     rebase_path("tools/ts_library_builder/main.ts", root_build_dir),
     "--basePath",
     rebase_path(".", root_build_dir),
+    "--inline",
+    rebase_path("js/lib.web_assembly.d.ts", root_build_dir),
     "--buildPath",
     rebase_path(root_build_dir, root_build_dir),
     "--outFile",

js/globals_test.ts

@@ -16,3 +16,7 @@ test(function windowWindowExists() {
 test(function globalThisEqualsWindow() {
   assert(globalThis === window);
 });
+
+test(function webAssemblyExists() {
+  assert(typeof WebAssembly.compile === "function");
+});

js/lib.web_assembly.d.ts

@@ -0,0 +1,163 @@
+// This follows the WebIDL at: https://webassembly.github.io/spec/js-api/
+// And follow on WebIDL at: https://webassembly.github.io/spec/web-api/
+
+declare namespace WebAssembly {
+  type WebAssemblyInstantiatedSource = {
+    module: Module;
+    instance: Instance;
+  };
+
+  /** Compiles a `WebAssembly.Module` from WebAssembly binary code.  This
+   * function is useful if it is necessary to a compile a module before it can
+   * be instantiated (otherwise, the `WebAssembly.instantiate()` function
+   * should be used). */
+  function compile(bufferSource: domTypes.BufferSource): Promise<Module>;
+
+  /** Compiles a `WebAssembly.Module` directly from a streamed underlying
+   * source. This function is useful if it is necessary to a compile a module
+   * before it can be instantiated (otherwise, the
+   * `WebAssembly.instantiateStreaming()` function should be used). */
+  function compileStreaming(
+    source: Promise<domTypes.Response>
+  ): Promise<Module>;
+
+  /** Takes the WebAssembly binary code, in the form of a typed array or
+   * `ArrayBuffer`, and performs both compilation and instantiation in one step.
+   * The returned `Promise` resolves to both a compiled `WebAssembly.Module` and
+   * its first `WebAssembly.Instance`. */
+  function instantiate(
+    bufferSource: domTypes.BufferSource,
+    importObject?: object
+  ): Promise<WebAssemblyInstantiatedSource>;
+
+  /** Takes an already-compiled `WebAssembly.Module` and returns a `Promise`
+   * that resolves to an `Instance` of that `Module`. This overload is useful if
+   * the `Module` has already been compiled. */
+  function instantiate(
+    module: Module,
+    importObject?: object
+  ): Promise<Instance>;
+
+  /** Compiles and instantiates a WebAssembly module directly from a streamed
+   * underlying source. This is the most efficient, optimized way to load wasm
+   * code. */
+  function instantiateStreaming(
+    source: Promise<domTypes.Response>,
+    importObject?: object
+  ): Promise<WebAssemblyInstantiatedSource>;
+
+  /** Validates a given typed array of WebAssembly binary code, returning
+   * whether the bytes form a valid wasm module (`true`) or not (`false`). */
+  function validate(bufferSource: domTypes.BufferSource): boolean;
+
+  type ImportExportKind = "function" | "table" | "memory" | "global";
+
+  type ModuleExportDescriptor = { name: string; kind: ImportExportKind };
+  type ModuleImportDescriptor = {
+    module: string;
+    name: string;
+    kind: ImportExportKind;
+  };
+
+  class Module {
+    constructor(bufferSource: domTypes.BufferSource);
+
+    /** Given a `Module` and string, returns a copy of the contents of all
+     * custom sections in the module with the given string name. */
+    static customSections(
+      moduleObject: Module,
+      sectionName: string
+    ): ArrayBuffer;
+
+    /** Given a `Module`, returns an array containing descriptions of all the
+     * declared exports. */
+    static exports(moduleObject: Module): ModuleExportDescriptor[];
+
+    /** Given a `Module`, returns an array containing descriptions of all the
+     * declared imports. */
+    static imports(moduleObject: Module): ModuleImportDescriptor[];
+  }
+
+  class Instance<T extends object = { [key: string]: any }> {
+    constructor(module: Module, importObject?: object);
+
+    /** An object containing as its members all the functions exported from the
+     * WebAssembly module instance, to allow them to be accessed and used by
+     * JavaScript. */
+    readonly exports: T;
+  }
+
+  type MemoryDescriptor = {
+    initial: number;
+    maximum?: number;
+  };
+
+  class Memory {
+    constructor(descriptor: MemoryDescriptor);
+
+    /** An accessor property that returns the buffer contained in the memory. */
+    readonly buffer: ArrayBuffer;
+
+    /** Increases the size of the memory instance by a specified number of
+     * WebAssembly pages (each one is 64KB in size). */
+    grow(delta: number): number;
+  }
+
+  type TableKind = "anyfunc";
+
+  interface TableDescriptor {
+    element: TableKind;
+    initial: number;
+    maximum?: number;
+  }
+
+  class Table {
+    constructor(descriptor: TableDescriptor);
+
+    /** Returns the length of the table, i.e. the number of elements. */
+    readonly length: number;
+
+    /** Accessor function — gets the element stored at a given index. */
+    get(index: number): (...args: any[]) => any;
+
+    /** Increases the size of the Table instance by a specified number of
+     * elements. */
+    grow(delta: number): number;
+
+    /** Sets an element stored at a given index to a given value. */
+    set(index: number, value: (...args: any[]) => any): void;
+  }
+
+  type GlobalDescriptor = { value: string; mutable?: boolean };
+
+  /** Represents a global variable instance, accessible from both JavaScript and
+   * importable/exportable across one or more `WebAssembly.Module` instances.
+   * This allows dynamic linking of multiple modules. */
+  class Global {
+    constructor(descriptor: GlobalDescriptor, value?: any);
+
+    /** Old-style method that returns the value contained inside the global
+     * variable. */
+    valueOf(): any;
+
+    /** The value contained inside the global variable — this can be used to
+     * directly set and get the global's value. */
+    value: any;
+  }
+
+  /** Indicates an error during WebAssembly decoding or validation */
+  class CompileError extends Error {
+    constructor(message: string, fileName?: string, lineNumber?: string);
+  }
+
+  /** Indicates an error during module instantiation (besides traps from the
+   * start function). */
+  class LinkError extends Error {
+    constructor(message: string, fileName?: string, lineNumber?: string);
+  }
+
+  /** Is thrown whenever WebAssembly specifies a trap. */
+  class RuntimeError extends Error {
+    constructor(message: string, fileName?: string, lineNumber?: string);
+  }
+}

tests/wasm.test

@@ -0,0 +1,2 @@
+args: tests/wasm.ts
+output: tests/wasm.ts.out

tests/wasm.ts

@@ -0,0 +1,15 @@
+// prettier-ignore
+const wasmCode = new Uint8Array([
+    0, 97, 115, 109, 1, 0, 0, 0, 1, 133, 128, 128, 128, 0, 1, 96, 0, 1, 127,
+    3, 130, 128, 128, 128, 0, 1, 0, 4, 132, 128, 128, 128, 0, 1, 112, 0, 0,
+    5, 131, 128, 128, 128, 0, 1, 0, 1, 6, 129, 128, 128, 128, 0, 0, 7, 145,
+    128, 128, 128, 0, 2, 6, 109, 101, 109, 111, 114, 121, 2, 0, 4, 109, 97,
+    105, 110, 0, 0, 10, 138, 128, 128, 128, 0, 1, 132, 128, 128, 128, 0, 0,
+    65, 42, 11
+  ]);
+
+const wasmModule = new WebAssembly.Module(wasmCode);
+
+const wasmInstance = new WebAssembly.Instance(wasmModule);
+
+console.log(wasmInstance.exports.main().toString());

tests/wasm.ts.out

@@ -0,0 +1 @@
+42

tools/ts_library_builder/ast_util.ts

@@ -243,6 +243,32 @@ export function getSourceComment(
   return `\n// @url ${relative(rootPath, sourceFile.getFilePath())}\n\n`;
 }
 
+interface InlineFilesOptions {
+  basePath: string;
+  debug?: boolean;
+  inline: string[];
+  targetSourceFile: SourceFile;
+}
+
+/** Inline files into the target source file. */
+export function inlineFiles({
+  basePath,
+  debug,
+  inline,
+  targetSourceFile
+}: InlineFilesOptions) {
+  for (const filename of inline) {
+    const text = readFileSync(filename, {
+      encoding: "utf8"
+    });
+    targetSourceFile.addStatements(
+      debug
+        ? `\n// @url ${relative(basePath, filename)}\n\n${text}`
+        : `\n${text}`
+    );
+  }
+}
+
 /**
  * Load and write to a virtual file system all the default libs needed to
  * resolve types on project.

tools/ts_library_builder/build_library.ts

@@ -16,6 +16,7 @@ import {
   checkDiagnostics,
   flattenNamespace,
   getSourceComment,
+  inlineFiles,
   loadDtsFiles,
   loadFiles,
   logDiagnostics,
@@ -41,6 +42,11 @@ export interface BuildLibraryOptions {
    */
   debug?: boolean;
 
+  /**
+   * An array of files that should be inlined into the library
+   */
+  inline?: string[];
+
   /**
    * The path to the output library
    */
@@ -278,6 +284,7 @@ export function mergeGlobal({
 export function main({
   basePath,
   buildPath,
+  inline,
   debug,
   outFile,
   silent
@@ -288,6 +295,12 @@ export function main({
     console.log();
     console.log(`basePath: "${basePath}"`);
     console.log(`buildPath: "${buildPath}"`);
+    if (inline && inline.length) {
+      console.log(`inline:`);
+      for (const filename of inline) {
+        console.log(`  "${filename}"`);
+      }
+    }
     console.log(`debug: ${!!debug}`);
     console.log(`outFile: "${outFile}"`);
     console.log();
@@ -431,6 +444,17 @@ export function main({
     console.log(`Merged "globals" into global scope.`);
   }
 
+  // Inline any files that were passed in, to be used to add additional libs
+  // which are not part of TypeScript.
+  if (inline && inline.length) {
+    inlineFiles({
+      basePath,
+      debug,
+      inline,
+      targetSourceFile: libDTs
+    });
+  }
+
   // Add the preamble
   libDTs.insertStatements(0, libPreamble);
 

tools/ts_library_builder/main.ts

@@ -7,6 +7,7 @@ import { main as buildRuntimeLib } from "./build_library";
 let basePath = process.cwd();
 let buildPath = path.join(basePath, "out", "debug");
 let outFile = path.join(buildPath, "gen", "lib", "lib.d.ts");
+let inline: string[] = [];
 let debug = false;
 let silent = false;
 
@@ -19,6 +20,11 @@ process.argv.forEach((arg, i, argv) => {
     case "--buildPath":
       buildPath = path.resolve(argv[i + 1]);
       break;
+    case "--inline":
+      inline = argv[i + 1].split(",").map(filename => {
+        return path.resolve(filename);
+      });
+      break;
     case "--outFile":
       outFile = path.resolve(argv[i + 1]);
       break;
@@ -35,6 +41,7 @@ buildRuntimeLib({
   basePath,
   buildPath,
   debug,
+  inline,
   outFile,
   silent
 });

tools/ts_library_builder/test.ts

@@ -11,7 +11,7 @@ import {
   test
 } from "../../js/deps/https/deno.land/x/std/testing/mod";
 import { flatten, mergeGlobal } from "./build_library";
-import { loadDtsFiles } from "./ast_util";
+import { inlineFiles, loadDtsFiles } from "./ast_util";
 
 const { ModuleKind, ModuleResolutionKind, ScriptTarget } = ts;
 
@@ -167,6 +167,26 @@ test(function buildLibraryMerge() {
   assertEqual(typeAliases.length, 1);
 });
 
+test(function testInlineFiles() {
+  const {
+    basePath,
+    buildPath,
+    debug,
+    outputSourceFile: targetSourceFile
+  } = setupFixtures();
+
+  inlineFiles({
+    basePath,
+    debug,
+    inline: [`${buildPath}/lib.extra.d.ts`],
+    targetSourceFile
+  });
+
+  assert(targetSourceFile.getNamespace("Qat") != null);
+  const qatNamespace = targetSourceFile.getNamespaceOrThrow("Qat");
+  assert(qatNamespace.getClass("Foo") != null);
+});
+
 // TODO author unit tests for `ast_util.ts`
 
 runTests();

tools/ts_library_builder/testdata/lib.extra.d.ts

@@ -0,0 +1,7 @@
+// comment
+
+declare namespace Qat {
+  class Foo {
+    bar: string;
+  }
+}

tsconfig.json

@@ -24,6 +24,7 @@
   },
   "files": [
     "node_modules/typescript/lib/lib.esnext.d.ts",
+    "js/lib.web_assembly.d.ts",
     "js/main.ts",
     "js/compiler.ts"
   ]