diff --git a/docs/contributing/web_platform_tests.md b/docs/contributing/web_platform_tests.md index 5d0774f3bb..9077655e9e 100644 --- a/docs/contributing/web_platform_tests.md +++ b/docs/contributing/web_platform_tests.md @@ -61,6 +61,16 @@ it. This will check that the python3 (or `python.exe` on Windows) is actually Python 3. +You can specify the following flags to customize bahaviour: + +``` +--rebuild + Rebuild the manifest instead of downloading. This can take up to 3 minutes. + +--auto-config + Automatically configure /etc/hosts if it is not configured (no prompt will be shown). +``` + #### `run` Run all tests like specified in `expectation.json`. diff --git a/docs/toc.json b/docs/toc.json index 6c6604ed28..0b740c07b7 100644 --- a/docs/toc.json +++ b/docs/toc.json @@ -93,6 +93,7 @@ "children": { "building_from_source": "Building from source", "development_tools": "Development tools", + "web_platform_tests": "Web platform tests", "style_guide": "Style guide", "architecture": "Architecture", "release_schedule": "Release schedule" diff --git a/op_crates/fetch/21_blob.js b/op_crates/fetch/21_blob.js new file mode 100644 index 0000000000..552441b21b --- /dev/null +++ b/op_crates/fetch/21_blob.js @@ -0,0 +1,294 @@ +// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. + +// @ts-check +/// +/// +/// +/// +/// +/// +/// + +((window) => { + // TODO(lucacasonato): this needs to not be hardcoded and instead depend on + // host os. + const isWindows = false; + + /** + * @param {string} input + * @param {number} position + * @returns {{result: string, position: number}} + */ + function collectCodepointsNotCRLF(input, position) { + // See https://w3c.github.io/FileAPI/#convert-line-endings-to-native and + // https://infra.spec.whatwg.org/#collect-a-sequence-of-code-points + const start = position; + for ( + let c = input.charAt(position); + position < input.length && !(c === "\r" || c === "\n"); + c = input.charAt(++position) + ); + return { result: input.slice(start, position), position }; + } + + /** + * @param {string} s + * @returns {string} + */ + function convertLineEndingsToNative(s) { + const nativeLineEnding = isWindows ? "\r\n" : "\n"; + + let { result, position } = collectCodepointsNotCRLF(s, 0); + + while (position < s.length) { + const codePoint = s.charAt(position); + if (codePoint === "\r") { + result += nativeLineEnding; + position++; + if (position < s.length && s.charAt(position) === "\n") { + position++; + } + } else if (codePoint === "\n") { + position++; + result += nativeLineEnding; + } + const { result: token, position: newPosition } = collectCodepointsNotCRLF( + s, + position, + ); + position = newPosition; + result += token; + } + + return result; + } + + /** + * @param {...Uint8Array} bytesArrays + * @returns {Uint8Array} + */ + function concatUint8Arrays(...bytesArrays) { + let byteLength = 0; + for (const bytes of bytesArrays) { + byteLength += bytes.byteLength; + } + const finalBytes = new Uint8Array(byteLength); + let current = 0; + for (const bytes of bytesArrays) { + finalBytes.set(bytes, current); + current += bytes.byteLength; + } + return finalBytes; + } + + const utf8Encoder = new TextEncoder(); + const utf8Decoder = new TextDecoder(); + + /** @typedef {BufferSource | Blob | string} BlobPart */ + + /** + * @param {BlobPart[]} parts + * @param {string} endings + * @returns {Uint8Array} + */ + function processBlobParts(parts, endings) { + /** @type {Uint8Array[]} */ + const bytesArrays = []; + for (const element of parts) { + if (element instanceof ArrayBuffer) { + bytesArrays.push(new Uint8Array(element.slice(0))); + } else if (ArrayBuffer.isView(element)) { + const buffer = element.buffer.slice( + element.byteOffset, + element.byteOffset + element.byteLength, + ); + bytesArrays.push(new Uint8Array(buffer)); + } else if (element instanceof Blob) { + bytesArrays.push( + new Uint8Array(element[_byteSequence].buffer.slice(0)), + ); + } else if (typeof element === "string") { + let s = element; + if (endings == "native") { + s = convertLineEndingsToNative(s); + } + bytesArrays.push(utf8Encoder.encode(s)); + } else { + throw new TypeError("Unreachable code (invalild element type)"); + } + } + return concatUint8Arrays(...bytesArrays); + } + + /** + * @param {string} str + * @returns {string} + */ + function normalizeType(str) { + let normalizedType = str; + if (!/^[\x20-\x7E]*$/.test(str)) { + normalizedType = ""; + } + return normalizedType.toLowerCase(); + } + + const _byteSequence = Symbol("[[ByteSequence]]"); + + class Blob { + /** @type {string} */ + #type; + + /** @type {Uint8Array} */ + [_byteSequence]; + + /** + * @param {BlobPart[]} [blobParts] + * @param {BlobPropertyBag} [options] + */ + constructor(blobParts, options) { + if (blobParts === undefined) { + blobParts = []; + } + if (typeof blobParts !== "object") { + throw new TypeError( + `Failed to construct 'Blob'. blobParts cannot be converted to a sequence.`, + ); + } + + const parts = []; + const iterator = blobParts[Symbol.iterator]?.(); + if (iterator === undefined) { + throw new TypeError( + "Failed to construct 'Blob'. The provided value cannot be converted to a sequence", + ); + } + while (true) { + const { value: element, done } = iterator.next(); + if (done) break; + if ( + ArrayBuffer.isView(element) || element instanceof ArrayBuffer || + element instanceof Blob + ) { + parts.push(element); + } else { + parts.push(String(element)); + } + } + + if (!options || typeof options === "function") { + options = {}; + } + if (typeof options !== "object") { + throw new TypeError( + `Failed to construct 'Blob'. options is not an object.`, + ); + } + const endings = options.endings?.toString() ?? "transparent"; + const type = options.type?.toString() ?? ""; + + /** @type {Uint8Array} */ + this[_byteSequence] = processBlobParts(parts, endings); + this.#type = normalizeType(type); + } + + /** @returns {number} */ + get size() { + return this[_byteSequence].byteLength; + } + + /** @returns {string} */ + get type() { + return this.#type; + } + + /** + * @param {number} [start] + * @param {number} [end] + * @param {string} [contentType] + * @returns {Blob} + */ + slice(start, end, contentType) { + const O = this; + /** @type {number} */ + let relativeStart; + if (start === undefined) { + relativeStart = 0; + } else { + start = Number(start); + if (start < 0) { + relativeStart = Math.max(O.size + start, 0); + } else { + relativeStart = Math.min(start, O.size); + } + } + /** @type {number} */ + let relativeEnd; + if (end === undefined) { + relativeEnd = O.size; + } else { + end = Number(end); + if (end < 0) { + relativeEnd = Math.max(O.size + end, 0); + } else { + relativeEnd = Math.min(end, O.size); + } + } + /** @type {string} */ + let relativeContentType; + if (contentType === undefined) { + relativeContentType = ""; + } else { + relativeContentType = normalizeType(String(contentType)); + } + return new Blob([ + O[_byteSequence].buffer.slice(relativeStart, relativeEnd), + ], { type: relativeContentType }); + } + + /** + * @returns {ReadableStream} + */ + stream() { + const bytes = this[_byteSequence]; + const stream = new ReadableStream({ + type: "bytes", + /** @param {ReadableByteStreamController} controller */ + start(controller) { + const chunk = new Uint8Array(bytes.buffer.slice(0)); + if (chunk.byteLength > 0) controller.enqueue(chunk); + controller.close(); + }, + }); + return stream; + } + + /** + * @returns {Promise} + */ + async text() { + const buffer = await this.arrayBuffer(); + return utf8Decoder.decode(buffer); + } + + /** + * @returns {Promise} + */ + async arrayBuffer() { + const stream = this.stream(); + let bytes = new Uint8Array(); + for await (const chunk of stream) { + bytes = concatUint8Arrays(bytes, chunk); + } + return bytes.buffer; + } + + get [Symbol.toStringTag]() { + return "Blob"; + } + } + + window.__bootstrap.blob = { + Blob, + _byteSequence, + }; +})(this); diff --git a/op_crates/fetch/26_fetch.js b/op_crates/fetch/26_fetch.js index 0176454670..47d701f3c9 100644 --- a/op_crates/fetch/26_fetch.js +++ b/op_crates/fetch/26_fetch.js @@ -21,11 +21,7 @@ window.__bootstrap.streams; const { DomIterableMixin } = window.__bootstrap.domIterable; const { Headers } = window.__bootstrap.headers; - - // FIXME(bartlomieju): stubbed out, needed in blob - const build = { - os: "", - }; + const { Blob, _byteSequence } = window.__bootstrap.blob; const MAX_SIZE = 2 ** 32 - 2; @@ -229,271 +225,6 @@ const LF = "\n".charCodeAt(0); const dataSymbol = Symbol("data"); - const bytesSymbol = Symbol("bytes"); - - /** - * @param {string} str - * @returns {boolean} - */ - function containsOnlyASCII(str) { - if (typeof str !== "string") { - return false; - } - // deno-lint-ignore no-control-regex - return /^[\x00-\x7F]*$/.test(str); - } - - /** - * @param {string} s - * @returns {string} - */ - function convertLineEndingsToNative(s) { - const nativeLineEnd = build.os == "windows" ? "\r\n" : "\n"; - - let position = 0; - - let collectionResult = collectSequenceNotCRLF(s, position); - - let token = collectionResult.collected; - position = collectionResult.newPosition; - - let result = token; - - while (position < s.length) { - const c = s.charAt(position); - if (c == "\r") { - result += nativeLineEnd; - position++; - if (position < s.length && s.charAt(position) == "\n") { - position++; - } - } else if (c == "\n") { - position++; - result += nativeLineEnd; - } - - collectionResult = collectSequenceNotCRLF(s, position); - - token = collectionResult.collected; - position = collectionResult.newPosition; - - result += token; - } - - return result; - } - - /** - * @param {string} s - * @param {number} position - * @returns {{ collected: string, newPosition: number }} - */ - function collectSequenceNotCRLF( - s, - position, - ) { - const start = position; - for ( - let c = s.charAt(position); - position < s.length && !(c == "\r" || c == "\n"); - c = s.charAt(++position) - ); - return { collected: s.slice(start, position), newPosition: position }; - } - - /** - * @param {BlobPart[]} blobParts - * @param {boolean} doNormalizeLineEndingsToNative - * @returns {Uint8Array[]} - */ - function toUint8Arrays( - blobParts, - doNormalizeLineEndingsToNative, - ) { - /** @type {Uint8Array[]} */ - const ret = []; - const enc = new TextEncoder(); - for (const element of blobParts) { - if (typeof element === "string") { - let str = element; - if (doNormalizeLineEndingsToNative) { - str = convertLineEndingsToNative(element); - } - ret.push(enc.encode(str)); - // eslint-disable-next-line @typescript-eslint/no-use-before-define - } else if (element instanceof Blob) { - ret.push(element[bytesSymbol]); - } else if (element instanceof Uint8Array) { - ret.push(element); - } else if (element instanceof Uint16Array) { - const uint8 = new Uint8Array(element.buffer); - ret.push(uint8); - } else if (element instanceof Uint32Array) { - const uint8 = new Uint8Array(element.buffer); - ret.push(uint8); - } else if (ArrayBuffer.isView(element)) { - // Convert view to Uint8Array. - const uint8 = new Uint8Array(element.buffer); - ret.push(uint8); - } else if (element instanceof ArrayBuffer) { - // Create a new Uint8Array view for the given ArrayBuffer. - const uint8 = new Uint8Array(element); - ret.push(uint8); - } else { - ret.push(enc.encode(String(element))); - } - } - return ret; - } - - /** - * @param {BlobPart[]} blobParts - * @param {BlobPropertyBag} options - * @returns {Uint8Array} - */ - function processBlobParts( - blobParts, - options, - ) { - const normalizeLineEndingsToNative = options.ending === "native"; - // ArrayBuffer.transfer is not yet implemented in V8, so we just have to - // pre compute size of the array buffer and do some sort of static allocation - // instead of dynamic allocation. - const uint8Arrays = toUint8Arrays(blobParts, normalizeLineEndingsToNative); - const byteLength = uint8Arrays - .map((u8) => u8.byteLength) - .reduce((a, b) => a + b, 0); - const ab = new ArrayBuffer(byteLength); - const bytes = new Uint8Array(ab); - let courser = 0; - for (const u8 of uint8Arrays) { - bytes.set(u8, courser); - courser += u8.byteLength; - } - - return bytes; - } - - /** - * @param {Uint8Array} blobBytes - */ - function getStream(blobBytes) { - // TODO(bartlomieju): Align to spec https://fetch.spec.whatwg.org/#concept-construct-readablestream - /** @type {ReadableStream} */ - return new ReadableStream({ - type: "bytes", - /** @param {ReadableStreamDefaultController} controller */ - start: (controller) => { - controller.enqueue(blobBytes); - controller.close(); - }, - }); - } - - /** @param {ReadableStreamReader} reader */ - async function readBytes( - reader, - ) { - const chunks = []; - while (true) { - const { done, value } = await reader.read(); - if (!done && value instanceof Uint8Array) { - chunks.push(value); - } else if (done) { - const size = chunks.reduce((p, i) => p + i.byteLength, 0); - const bytes = new Uint8Array(size); - let offs = 0; - for (const chunk of chunks) { - bytes.set(chunk, offs); - offs += chunk.byteLength; - } - return bytes.buffer; - } else { - throw new TypeError("Invalid reader result."); - } - } - } - - // A WeakMap holding blob to byte array mapping. - // Ensures it does not impact garbage collection. - // const blobBytesWeakMap = new WeakMap(); - - class Blob { - /** @type {number} */ - size = 0; - /** @type {string} */ - type = ""; - - /** - * - * @param {BlobPart[]} blobParts - * @param {BlobPropertyBag | undefined} options - */ - constructor(blobParts, options) { - if (arguments.length === 0) { - this[bytesSymbol] = new Uint8Array(); - return; - } - - const { ending = "transparent", type = "" } = options ?? {}; - // Normalize options.type. - let normalizedType = type; - if (!containsOnlyASCII(type)) { - normalizedType = ""; - } else { - if (type.length) { - for (let i = 0; i < type.length; ++i) { - const char = type[i]; - if (char < "\u0020" || char > "\u007E") { - normalizedType = ""; - break; - } - } - normalizedType = type.toLowerCase(); - } - } - const bytes = processBlobParts(blobParts, { ending, type }); - // Set Blob object's properties. - this[bytesSymbol] = bytes; - this.size = bytes.byteLength; - this.type = normalizedType; - } - - /** - * @param {number} start - * @param {number} end - * @param {string} contentType - * @returns {Blob} - */ - slice(start, end, contentType) { - return new Blob([this[bytesSymbol].slice(start, end)], { - type: contentType || this.type, - }); - } - - /** - * @returns {ReadableStream} - */ - stream() { - return getStream(this[bytesSymbol]); - } - - /** - * @returns {Promise} - */ - async text() { - const reader = getStream(this[bytesSymbol]).getReader(); - const decoder = new TextDecoder(); - return decoder.decode(await readBytes(reader)); - } - - /** - * @returns {Promise} - */ - arrayBuffer() { - return readBytes(getStream(this[bytesSymbol]).getReader()); - } - } class DomFile extends Blob { /** @@ -761,7 +492,7 @@ */ #writeFile = (field, value) => { this.#writeFileHeaders(field, value.name, value.type); - this.writer.writeSync(value[bytesSymbol]); + this.writer.writeSync(value[_byteSequence]); }; } @@ -1607,7 +1338,7 @@ body = new TextEncoder().encode(init.body.toString()); contentType = "application/x-www-form-urlencoded;charset=UTF-8"; } else if (init.body instanceof Blob) { - body = init.body[bytesSymbol]; + body = init.body[_byteSequence]; contentType = init.body.type; } else if (init.body instanceof FormData) { let boundary; @@ -1762,7 +1493,6 @@ } window.__bootstrap.fetch = { - Blob, File: DomFile, FormData, setBaseUrl, diff --git a/op_crates/fetch/internal.d.ts b/op_crates/fetch/internal.d.ts index e02bc6ed2b..5fb30f5034 100644 --- a/op_crates/fetch/internal.d.ts +++ b/op_crates/fetch/internal.d.ts @@ -19,6 +19,13 @@ declare namespace globalThis { Headers: typeof Headers; }; + declare var blob: { + Blob: typeof Blob & { + [globalThis.__bootstrap.blob._byteSequence]: Uint8Array; + }; + _byteSequence: unique symbol; + }; + declare var streams: { ReadableStream: typeof ReadableStream; isReadableStreamDisturbed(stream: ReadableStream): boolean; diff --git a/op_crates/fetch/lib.deno_fetch.d.ts b/op_crates/fetch/lib.deno_fetch.d.ts index f18dbe359e..7d06fe691d 100644 --- a/op_crates/fetch/lib.deno_fetch.d.ts +++ b/op_crates/fetch/lib.deno_fetch.d.ts @@ -291,7 +291,7 @@ type BlobPart = BufferSource | Blob | string; interface BlobPropertyBag { type?: string; - ending?: "transparent" | "native"; + endings?: "transparent" | "native"; } /** A file-like object of immutable, raw data. Blobs represent data that isn't necessarily in a JavaScript-native format. The File interface is based on Blob, inheriting blob functionality and expanding it to support files on the user's system. */ diff --git a/op_crates/fetch/lib.rs b/op_crates/fetch/lib.rs index 256e7904da..46c72e7a88 100644 --- a/op_crates/fetch/lib.rs +++ b/op_crates/fetch/lib.rs @@ -65,6 +65,10 @@ pub fn init(isolate: &mut JsRuntime) { "deno:op_crates/fetch/20_headers.js", include_str!("20_headers.js"), ), + ( + "deno:op_crates/fetch/21_blob.js", + include_str!("21_blob.js"), + ), ( "deno:op_crates/fetch/26_fetch.js", include_str!("26_fetch.js"), diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index e227c3ecac..c3ca7b772c 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -27,6 +27,7 @@ delete Object.prototype.__proto__; const streams = window.__bootstrap.streams; const fileReader = window.__bootstrap.fileReader; const webSocket = window.__bootstrap.webSocket; + const blob = window.__bootstrap.blob; const fetch = window.__bootstrap.fetch; const prompt = window.__bootstrap.prompt; const denoNs = window.__bootstrap.denoNs; @@ -197,7 +198,7 @@ delete Object.prototype.__proto__; // https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope const windowOrWorkerGlobalScope = { - Blob: util.nonEnumerable(fetch.Blob), + Blob: util.nonEnumerable(blob.Blob), ByteLengthQueuingStrategy: util.nonEnumerable( streams.ByteLengthQueuingStrategy, ), diff --git a/test_util/wpt b/test_util/wpt index ec40449a41..4e9ee672ed 160000 --- a/test_util/wpt +++ b/test_util/wpt @@ -1 +1 @@ -Subproject commit ec40449a41939504a6adc039e7d98f52ec8894c9 +Subproject commit 4e9ee672edb2764c51904739bf8397b959b3a85c diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json index b8a02d5a3e..307021cd8e 100644 --- a/tools/wpt/expectation.json +++ b/tools/wpt/expectation.json @@ -1018,5 +1018,18 @@ ] } } + }, + "FileAPI": { + "blob": { + "Blob-array-buffer.any.js": true, + "Blob-stream.any.js": true, + "Blob-text.any.js": true, + "Blob-constructor.any.js": [ + "Blob interface object", + "Passing a FrozenArray as the blobParts array should work (FrozenArray)." + ], + "Blob-slice-overflow.any.js": true, + "Blob-slice.any.js": true + } } } \ No newline at end of file diff --git a/tools/wpt/utils.ts b/tools/wpt/utils.ts index 3efd252b4b..9ef07dec3d 100644 --- a/tools/wpt/utils.ts +++ b/tools/wpt/utils.ts @@ -7,6 +7,7 @@ export const { json, quiet, release, + rebuild, ["--"]: rest, ["auto-config"]: autoConfig, } = parse(Deno.args, { @@ -43,7 +44,15 @@ const MANIFEST_PATH = join(ROOT_PATH, "./tools/wpt/manifest.json"); export async function updateManifest() { const proc = runPy( - ["wpt", "manifest", "--tests-root", ".", "-p", MANIFEST_PATH], + [ + "wpt", + "manifest", + "--tests-root", + ".", + "-p", + MANIFEST_PATH, + ...(rebuild ? ["--rebuild"] : []), + ], {}, ); const status = await proc.status();