refactor: rewrite Blob implementation (#9309)

Co-authored-by: Kitson Kelly <me@kitsonkelly.com>

docs/contributing/web_platform_tests.md

@@ -61,6 +61,16 @@ it.
 This will check that the python3 (or `python.exe` on Windows) is actually
 Python 3.
 
+You can specify the following flags to customize bahaviour:
+
+```
+--rebuild
+    Rebuild the manifest instead of downloading. This can take up to 3 minutes.
+
+--auto-config
+    Automatically configure /etc/hosts if it is not configured (no prompt will be shown).
+```
+
 #### `run`
 
 Run all tests like specified in `expectation.json`.

docs/toc.json

@@ -93,6 +93,7 @@
     "children": {
       "building_from_source": "Building from source",
       "development_tools": "Development tools",
+      "web_platform_tests": "Web platform tests",
       "style_guide": "Style guide",
       "architecture": "Architecture",
       "release_schedule": "Release schedule"

op_crates/fetch/21_blob.js

@@ -0,0 +1,294 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+// @ts-check
+/// <reference no-default-lib="true" />
+/// <reference path="../../core/lib.deno_core.d.ts" />
+/// <reference path="../web/internal.d.ts" />
+/// <reference path="../web/lib.deno_web.d.ts" />
+/// <reference path="./internal.d.ts" />
+/// <reference path="./lib.deno_fetch.d.ts" />
+/// <reference lib="esnext" />
+
+((window) => {
+  // TODO(lucacasonato): this needs to not be hardcoded and instead depend on
+  // host os.
+  const isWindows = false;
+
+  /**
+   * @param {string} input
+   * @param {number} position
+   * @returns {{result: string, position: number}}
+   */
+  function collectCodepointsNotCRLF(input, position) {
+    // See https://w3c.github.io/FileAPI/#convert-line-endings-to-native and
+    // https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points
+    const start = position;
+    for (
+      let c = input.charAt(position);
+      position < input.length && !(c === "\r" || c === "\n");
+      c = input.charAt(++position)
+    );
+    return { result: input.slice(start, position), position };
+  }
+
+  /**
+   * @param {string} s
+   * @returns {string}
+   */
+  function convertLineEndingsToNative(s) {
+    const nativeLineEnding = isWindows ? "\r\n" : "\n";
+
+    let { result, position } = collectCodepointsNotCRLF(s, 0);
+
+    while (position < s.length) {
+      const codePoint = s.charAt(position);
+      if (codePoint === "\r") {
+        result += nativeLineEnding;
+        position++;
+        if (position < s.length && s.charAt(position) === "\n") {
+          position++;
+        }
+      } else if (codePoint === "\n") {
+        position++;
+        result += nativeLineEnding;
+      }
+      const { result: token, position: newPosition } = collectCodepointsNotCRLF(
+        s,
+        position,
+      );
+      position = newPosition;
+      result += token;
+    }
+
+    return result;
+  }
+
+  /**
+   * @param  {...Uint8Array} bytesArrays
+   * @returns {Uint8Array} 
+   */
+  function concatUint8Arrays(...bytesArrays) {
+    let byteLength = 0;
+    for (const bytes of bytesArrays) {
+      byteLength += bytes.byteLength;
+    }
+    const finalBytes = new Uint8Array(byteLength);
+    let current = 0;
+    for (const bytes of bytesArrays) {
+      finalBytes.set(bytes, current);
+      current += bytes.byteLength;
+    }
+    return finalBytes;
+  }
+
+  const utf8Encoder = new TextEncoder();
+  const utf8Decoder = new TextDecoder();
+
+  /** @typedef {BufferSource | Blob | string} BlobPart */
+
+  /** 
+     * @param {BlobPart[]} parts
+     * @param {string} endings
+     * @returns {Uint8Array}
+     */
+  function processBlobParts(parts, endings) {
+    /** @type {Uint8Array[]} */
+    const bytesArrays = [];
+    for (const element of parts) {
+      if (element instanceof ArrayBuffer) {
+        bytesArrays.push(new Uint8Array(element.slice(0)));
+      } else if (ArrayBuffer.isView(element)) {
+        const buffer = element.buffer.slice(
+          element.byteOffset,
+          element.byteOffset + element.byteLength,
+        );
+        bytesArrays.push(new Uint8Array(buffer));
+      } else if (element instanceof Blob) {
+        bytesArrays.push(
+          new Uint8Array(element[_byteSequence].buffer.slice(0)),
+        );
+      } else if (typeof element === "string") {
+        let s = element;
+        if (endings == "native") {
+          s = convertLineEndingsToNative(s);
+        }
+        bytesArrays.push(utf8Encoder.encode(s));
+      } else {
+        throw new TypeError("Unreachable code (invalild element type)");
+      }
+    }
+    return concatUint8Arrays(...bytesArrays);
+  }
+
+  /**
+   * @param {string} str 
+   * @returns {string}
+   */
+  function normalizeType(str) {
+    let normalizedType = str;
+    if (!/^[\x20-\x7E]*$/.test(str)) {
+      normalizedType = "";
+    }
+    return normalizedType.toLowerCase();
+  }
+
+  const _byteSequence = Symbol("[[ByteSequence]]");
+
+  class Blob {
+    /** @type {string} */
+    #type;
+
+    /** @type {Uint8Array} */
+    [_byteSequence];
+
+    /** 
+     * @param {BlobPart[]} [blobParts]
+     * @param {BlobPropertyBag} [options]
+     */
+    constructor(blobParts, options) {
+      if (blobParts === undefined) {
+        blobParts = [];
+      }
+      if (typeof blobParts !== "object") {
+        throw new TypeError(
+          `Failed to construct 'Blob'. blobParts cannot be converted to a sequence.`,
+        );
+      }
+
+      const parts = [];
+      const iterator = blobParts[Symbol.iterator]?.();
+      if (iterator === undefined) {
+        throw new TypeError(
+          "Failed to construct 'Blob'. The provided value cannot be converted to a sequence",
+        );
+      }
+      while (true) {
+        const { value: element, done } = iterator.next();
+        if (done) break;
+        if (
+          ArrayBuffer.isView(element) || element instanceof ArrayBuffer ||
+          element instanceof Blob
+        ) {
+          parts.push(element);
+        } else {
+          parts.push(String(element));
+        }
+      }
+
+      if (!options || typeof options === "function") {
+        options = {};
+      }
+      if (typeof options !== "object") {
+        throw new TypeError(
+          `Failed to construct 'Blob'. options is not an object.`,
+        );
+      }
+      const endings = options.endings?.toString() ?? "transparent";
+      const type = options.type?.toString() ?? "";
+
+      /** @type {Uint8Array} */
+      this[_byteSequence] = processBlobParts(parts, endings);
+      this.#type = normalizeType(type);
+    }
+
+    /** @returns {number} */
+    get size() {
+      return this[_byteSequence].byteLength;
+    }
+
+    /** @returns {string} */
+    get type() {
+      return this.#type;
+    }
+
+    /** 
+     * @param {number} [start]
+     * @param {number} [end]
+     * @param {string} [contentType]
+     * @returns {Blob}
+     */
+    slice(start, end, contentType) {
+      const O = this;
+      /** @type {number} */
+      let relativeStart;
+      if (start === undefined) {
+        relativeStart = 0;
+      } else {
+        start = Number(start);
+        if (start < 0) {
+          relativeStart = Math.max(O.size + start, 0);
+        } else {
+          relativeStart = Math.min(start, O.size);
+        }
+      }
+      /** @type {number} */
+      let relativeEnd;
+      if (end === undefined) {
+        relativeEnd = O.size;
+      } else {
+        end = Number(end);
+        if (end < 0) {
+          relativeEnd = Math.max(O.size + end, 0);
+        } else {
+          relativeEnd = Math.min(end, O.size);
+        }
+      }
+      /** @type {string} */
+      let relativeContentType;
+      if (contentType === undefined) {
+        relativeContentType = "";
+      } else {
+        relativeContentType = normalizeType(String(contentType));
+      }
+      return new Blob([
+        O[_byteSequence].buffer.slice(relativeStart, relativeEnd),
+      ], { type: relativeContentType });
+    }
+
+    /**
+     * @returns {ReadableStream<Uint8Array>}
+     */
+    stream() {
+      const bytes = this[_byteSequence];
+      const stream = new ReadableStream({
+        type: "bytes",
+        /** @param {ReadableByteStreamController} controller */
+        start(controller) {
+          const chunk = new Uint8Array(bytes.buffer.slice(0));
+          if (chunk.byteLength > 0) controller.enqueue(chunk);
+          controller.close();
+        },
+      });
+      return stream;
+    }
+
+    /**
+     * @returns {Promise<string>}
+     */
+    async text() {
+      const buffer = await this.arrayBuffer();
+      return utf8Decoder.decode(buffer);
+    }
+
+    /**
+     * @returns {Promise<ArrayBuffer>}
+     */
+    async arrayBuffer() {
+      const stream = this.stream();
+      let bytes = new Uint8Array();
+      for await (const chunk of stream) {
+        bytes = concatUint8Arrays(bytes, chunk);
+      }
+      return bytes.buffer;
+    }
+
+    get [Symbol.toStringTag]() {
+      return "Blob";
+    }
+  }
+
+  window.__bootstrap.blob = {
+    Blob,
+    _byteSequence,
+  };
+})(this);

op_crates/fetch/26_fetch.js

@@ -21,11 +21,7 @@
     window.__bootstrap.streams;
   const { DomIterableMixin } = window.__bootstrap.domIterable;
   const { Headers } = window.__bootstrap.headers;
-
+  const { Blob, _byteSequence } = window.__bootstrap.blob;
-  // FIXME(bartlomieju): stubbed out, needed in blob
-  const build = {
-    os: "",
-  };
 
   const MAX_SIZE = 2 ** 32 - 2;
 
@@ -229,271 +225,6 @@
   const LF = "\n".charCodeAt(0);
 
   const dataSymbol = Symbol("data");
-  const bytesSymbol = Symbol("bytes");
-
-  /**
-   * @param {string} str
-   * @returns {boolean}
-   */
-  function containsOnlyASCII(str) {
-    if (typeof str !== "string") {
-      return false;
-    }
-    // deno-lint-ignore no-control-regex
-    return /^[\x00-\x7F]*$/.test(str);
-  }
-
-  /**
-   * @param {string} s
-   * @returns {string}
-   */
-  function convertLineEndingsToNative(s) {
-    const nativeLineEnd = build.os == "windows" ? "\r\n" : "\n";
-
-    let position = 0;
-
-    let collectionResult = collectSequenceNotCRLF(s, position);
-
-    let token = collectionResult.collected;
-    position = collectionResult.newPosition;
-
-    let result = token;
-
-    while (position < s.length) {
-      const c = s.charAt(position);
-      if (c == "\r") {
-        result += nativeLineEnd;
-        position++;
-        if (position < s.length && s.charAt(position) == "\n") {
-          position++;
-        }
-      } else if (c == "\n") {
-        position++;
-        result += nativeLineEnd;
-      }
-
-      collectionResult = collectSequenceNotCRLF(s, position);
-
-      token = collectionResult.collected;
-      position = collectionResult.newPosition;
-
-      result += token;
-    }
-
-    return result;
-  }
-
-  /**
-   * @param {string} s
-   * @param {number} position
-   * @returns {{ collected: string, newPosition: number }}
-   */
-  function collectSequenceNotCRLF(
-    s,
-    position,
-  ) {
-    const start = position;
-    for (
-      let c = s.charAt(position);
-      position < s.length && !(c == "\r" || c == "\n");
-      c = s.charAt(++position)
-    );
-    return { collected: s.slice(start, position), newPosition: position };
-  }
-
-  /**
-   * @param {BlobPart[]} blobParts 
-   * @param {boolean} doNormalizeLineEndingsToNative
-   * @returns {Uint8Array[]}
-   */
-  function toUint8Arrays(
-    blobParts,
-    doNormalizeLineEndingsToNative,
-  ) {
-    /** @type {Uint8Array[]} */
-    const ret = [];
-    const enc = new TextEncoder();
-    for (const element of blobParts) {
-      if (typeof element === "string") {
-        let str = element;
-        if (doNormalizeLineEndingsToNative) {
-          str = convertLineEndingsToNative(element);
-        }
-        ret.push(enc.encode(str));
-        // eslint-disable-next-line @typescript-eslint/no-use-before-define
-      } else if (element instanceof Blob) {
-        ret.push(element[bytesSymbol]);
-      } else if (element instanceof Uint8Array) {
-        ret.push(element);
-      } else if (element instanceof Uint16Array) {
-        const uint8 = new Uint8Array(element.buffer);
-        ret.push(uint8);
-      } else if (element instanceof Uint32Array) {
-        const uint8 = new Uint8Array(element.buffer);
-        ret.push(uint8);
-      } else if (ArrayBuffer.isView(element)) {
-        // Convert view to Uint8Array.
-        const uint8 = new Uint8Array(element.buffer);
-        ret.push(uint8);
-      } else if (element instanceof ArrayBuffer) {
-        // Create a new Uint8Array view for the given ArrayBuffer.
-        const uint8 = new Uint8Array(element);
-        ret.push(uint8);
-      } else {
-        ret.push(enc.encode(String(element)));
-      }
-    }
-    return ret;
-  }
-
-  /**
-   * @param {BlobPart[]} blobParts 
-   * @param {BlobPropertyBag} options
-   * @returns {Uint8Array}
-   */
-  function processBlobParts(
-    blobParts,
-    options,
-  ) {
-    const normalizeLineEndingsToNative = options.ending === "native";
-    // ArrayBuffer.transfer is not yet implemented in V8, so we just have to
-    // pre compute size of the array buffer and do some sort of static allocation
-    // instead of dynamic allocation.
-    const uint8Arrays = toUint8Arrays(blobParts, normalizeLineEndingsToNative);
-    const byteLength = uint8Arrays
-      .map((u8) => u8.byteLength)
-      .reduce((a, b) => a + b, 0);
-    const ab = new ArrayBuffer(byteLength);
-    const bytes = new Uint8Array(ab);
-    let courser = 0;
-    for (const u8 of uint8Arrays) {
-      bytes.set(u8, courser);
-      courser += u8.byteLength;
-    }
-
-    return bytes;
-  }
-
-  /**
-   * @param {Uint8Array} blobBytes 
-   */
-  function getStream(blobBytes) {
-    // TODO(bartlomieju): Align to spec https://fetch.spec.whatwg.org/#concept-construct-readablestream
-    /** @type {ReadableStream<Uint8Array>} */
-    return new ReadableStream({
-      type: "bytes",
-      /** @param {ReadableStreamDefaultController} controller */
-      start: (controller) => {
-        controller.enqueue(blobBytes);
-        controller.close();
-      },
-    });
-  }
-
-  /** @param {ReadableStreamReader<Uint8Array>} reader */
-  async function readBytes(
-    reader,
-  ) {
-    const chunks = [];
-    while (true) {
-      const { done, value } = await reader.read();
-      if (!done && value instanceof Uint8Array) {
-        chunks.push(value);
-      } else if (done) {
-        const size = chunks.reduce((p, i) => p + i.byteLength, 0);
-        const bytes = new Uint8Array(size);
-        let offs = 0;
-        for (const chunk of chunks) {
-          bytes.set(chunk, offs);
-          offs += chunk.byteLength;
-        }
-        return bytes.buffer;
-      } else {
-        throw new TypeError("Invalid reader result.");
-      }
-    }
-  }
-
-  // A WeakMap holding blob to byte array mapping.
-  // Ensures it does not impact garbage collection.
-  // const blobBytesWeakMap = new WeakMap();
-
-  class Blob {
-    /** @type {number} */
-    size = 0;
-    /** @type {string} */
-    type = "";
-
-    /**
-     * 
-     * @param {BlobPart[]} blobParts 
-     * @param {BlobPropertyBag | undefined} options 
-     */
-    constructor(blobParts, options) {
-      if (arguments.length === 0) {
-        this[bytesSymbol] = new Uint8Array();
-        return;
-      }
-
-      const { ending = "transparent", type = "" } = options ?? {};
-      // Normalize options.type.
-      let normalizedType = type;
-      if (!containsOnlyASCII(type)) {
-        normalizedType = "";
-      } else {
-        if (type.length) {
-          for (let i = 0; i < type.length; ++i) {
-            const char = type[i];
-            if (char < "\u0020" || char > "\u007E") {
-              normalizedType = "";
-              break;
-            }
-          }
-          normalizedType = type.toLowerCase();
-        }
-      }
-      const bytes = processBlobParts(blobParts, { ending, type });
-      // Set Blob object's properties.
-      this[bytesSymbol] = bytes;
-      this.size = bytes.byteLength;
-      this.type = normalizedType;
-    }
-
-    /**
-     * @param {number} start 
-     * @param {number} end 
-     * @param {string} contentType 
-     * @returns {Blob}
-     */
-    slice(start, end, contentType) {
-      return new Blob([this[bytesSymbol].slice(start, end)], {
-        type: contentType || this.type,
-      });
-    }
-
-    /**
-     * @returns {ReadableStream<Uint8Array>}
-     */
-    stream() {
-      return getStream(this[bytesSymbol]);
-    }
-
-    /**
-     * @returns {Promise<string>}
-     */
-    async text() {
-      const reader = getStream(this[bytesSymbol]).getReader();
-      const decoder = new TextDecoder();
-      return decoder.decode(await readBytes(reader));
-    }
-
-    /**
-     * @returns {Promise<ArrayBuffer>}
-     */
-    arrayBuffer() {
-      return readBytes(getStream(this[bytesSymbol]).getReader());
-    }
-  }
 
   class DomFile extends Blob {
     /**
@@ -761,7 +492,7 @@
      */
     #writeFile = (field, value) => {
       this.#writeFileHeaders(field, value.name, value.type);
-      this.writer.writeSync(value[bytesSymbol]);
+      this.writer.writeSync(value[_byteSequence]);
     };
   }
 
@@ -1607,7 +1338,7 @@
             body = new TextEncoder().encode(init.body.toString());
             contentType = "application/x-www-form-urlencoded;charset=UTF-8";
           } else if (init.body instanceof Blob) {
-            body = init.body[bytesSymbol];
+            body = init.body[_byteSequence];
             contentType = init.body.type;
           } else if (init.body instanceof FormData) {
             let boundary;
@@ -1762,7 +1493,6 @@
   }
 
   window.__bootstrap.fetch = {
-    Blob,
     File: DomFile,
     FormData,
     setBaseUrl,

op_crates/fetch/internal.d.ts

@@ -19,6 +19,13 @@ declare namespace globalThis {
       Headers: typeof Headers;
     };
 
+    declare var blob: {
+      Blob: typeof Blob & {
+        [globalThis.__bootstrap.blob._byteSequence]: Uint8Array;
+      };
+      _byteSequence: unique symbol;
+    };
+
     declare var streams: {
       ReadableStream: typeof ReadableStream;
       isReadableStreamDisturbed(stream: ReadableStream): boolean;

op_crates/fetch/lib.deno_fetch.d.ts

@@ -291,7 +291,7 @@ type BlobPart = BufferSource | Blob | string;
 
 interface BlobPropertyBag {
   type?: string;
-  ending?: "transparent" | "native";
+  endings?: "transparent" | "native";
 }
 
 /** A file-like object of immutable, raw data. Blobs represent data that isn't necessarily in a JavaScript-native format. The File interface is based on Blob, inheriting blob functionality and expanding it to support files on the user's system. */

op_crates/fetch/lib.rs

@@ -65,6 +65,10 @@ pub fn init(isolate: &mut JsRuntime) {
       "deno:op_crates/fetch/20_headers.js",
       include_str!("20_headers.js"),
     ),
+    (
+      "deno:op_crates/fetch/21_blob.js",
+      include_str!("21_blob.js"),
+    ),
     (
       "deno:op_crates/fetch/26_fetch.js",
       include_str!("26_fetch.js"),

runtime/js/99_main.js

@@ -27,6 +27,7 @@ delete Object.prototype.__proto__;
   const streams = window.__bootstrap.streams;
   const fileReader = window.__bootstrap.fileReader;
   const webSocket = window.__bootstrap.webSocket;
+  const blob = window.__bootstrap.blob;
   const fetch = window.__bootstrap.fetch;
   const prompt = window.__bootstrap.prompt;
   const denoNs = window.__bootstrap.denoNs;
@@ -197,7 +198,7 @@ delete Object.prototype.__proto__;
 
   // https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope
   const windowOrWorkerGlobalScope = {
-    Blob: util.nonEnumerable(fetch.Blob),
+    Blob: util.nonEnumerable(blob.Blob),
     ByteLengthQueuingStrategy: util.nonEnumerable(
       streams.ByteLengthQueuingStrategy,
     ),

test_util/wpt

@@ -1 +1 @@
-Subproject commit ec40449a41939504a6adc039e7d98f52ec8894c9
+Subproject commit 4e9ee672edb2764c51904739bf8397b959b3a85c

tools/wpt/expectation.json

@@ -1018,5 +1018,18 @@
         ]
       }
     }
+  },
+  "FileAPI": {
+    "blob": {
+      "Blob-array-buffer.any.js": true,
+      "Blob-stream.any.js": true,
+      "Blob-text.any.js": true,
+      "Blob-constructor.any.js": [
+        "Blob interface object",
+        "Passing a FrozenArray as the blobParts array should work (FrozenArray<MessagePort>)."
+      ],
+      "Blob-slice-overflow.any.js": true,
+      "Blob-slice.any.js": true
+    }
   }
 }

tools/wpt/utils.ts

@@ -7,6 +7,7 @@ export const {
   json,
   quiet,
   release,
+  rebuild,
   ["--"]: rest,
   ["auto-config"]: autoConfig,
 } = parse(Deno.args, {
@@ -43,7 +44,15 @@ const MANIFEST_PATH = join(ROOT_PATH, "./tools/wpt/manifest.json");
 
 export async function updateManifest() {
   const proc = runPy(
-    ["wpt", "manifest", "--tests-root", ".", "-p", MANIFEST_PATH],
+    [
+      "wpt",
+      "manifest",
+      "--tests-root",
+      ".",
+      "-p",
+      MANIFEST_PATH,
+      ...(rebuild ? ["--rebuild"] : []),
+    ],
     {},
   );
   const status = await proc.status();