From 62bf4031576ab833a8057c6b7037e6476d13bf65 Mon Sep 17 00:00:00 2001 From: Andreu Botella Date: Sun, 6 Jun 2021 03:23:16 +0200 Subject: [PATCH] feat(web): Implement TextDecoderStream and TextEncoderStream (#10842) --- extensions/web/08_text_encoding.js | 194 +++++++++++++++++++++++++++++ extensions/web/lib.deno_web.d.ts | 25 +++- runtime/js/99_main.js | 2 + tools/wpt/expectation.json | 45 +++---- 4 files changed, 234 insertions(+), 32 deletions(-) diff --git a/extensions/web/08_text_encoding.js b/extensions/web/08_text_encoding.js index 7b2c974974..be66e4981b 100644 --- a/extensions/web/08_text_encoding.js +++ b/extensions/web/08_text_encoding.js @@ -3,6 +3,7 @@ // @ts-check /// /// +/// /// /// /// @@ -203,6 +204,197 @@ configurable: true, }); + class TextDecoderStream { + /** @type {TextDecoder} */ + #decoder; + /** @type {TransformStream} */ + #transform; + + /** + * + * @param {string} label + * @param {TextDecoderOptions} options + */ + constructor(label = "utf-8", options = {}) { + const prefix = "Failed to construct 'TextDecoderStream'"; + label = webidl.converters.DOMString(label, { + prefix, + context: "Argument 1", + }); + options = webidl.converters.TextDecoderOptions(options, { + prefix, + context: "Argument 2", + }); + this.#decoder = new TextDecoder(label, options); + this.#transform = new TransformStream({ + // The transform and flush functions need access to TextDecoderStream's + // `this`, so they are defined as functions rather than methods. + transform: (chunk, controller) => { + try { + chunk = webidl.converters.BufferSource(chunk, { + allowShared: true, + }); + const decoded = this.#decoder.decode(chunk, { stream: true }); + if (decoded) { + controller.enqueue(decoded); + } + return Promise.resolve(); + } catch (err) { + return Promise.reject(err); + } + }, + flush: (controller) => { + try { + const final = this.#decoder.decode(); + if (final) { + controller.enqueue(final); + } + return Promise.resolve(); + } catch (err) { + return Promise.reject(err); + } + }, + }); + this[webidl.brand] = webidl.brand; + } + + /** @returns {string} */ + get encoding() { + webidl.assertBranded(this, TextDecoderStream); + return this.#decoder.encoding; + } + + /** @returns {boolean} */ + get fatal() { + webidl.assertBranded(this, TextDecoderStream); + return this.#decoder.fatal; + } + + /** @returns {boolean} */ + get ignoreBOM() { + webidl.assertBranded(this, TextDecoderStream); + return this.#decoder.ignoreBOM; + } + + /** @returns {ReadableStream} */ + get readable() { + webidl.assertBranded(this, TextDecoderStream); + return this.#transform.readable; + } + + /** @returns {WritableStream} */ + get writable() { + webidl.assertBranded(this, TextDecoderStream); + return this.#transform.writable; + } + + get [Symbol.toStringTag]() { + return "TextDecoderStream"; + } + } + + Object.defineProperty(TextDecoderStream.prototype, "encoding", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextDecoderStream.prototype, "fatal", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextDecoderStream.prototype, "ignoreBOM", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextDecoderStream.prototype, "readable", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextDecoderStream.prototype, "writable", { + enumerable: true, + configurable: true, + }); + + class TextEncoderStream { + /** @type {string | null} */ + #pendingHighSurrogate = null; + /** @type {TransformStream} */ + #transform; + + constructor() { + this.#transform = new TransformStream({ + // The transform and flush functions need access to TextEncoderStream's + // `this`, so they are defined as functions rather than methods. + transform: (chunk, controller) => { + try { + chunk = webidl.converters.DOMString(chunk); + if (this.#pendingHighSurrogate !== null) { + chunk = this.#pendingHighSurrogate + chunk; + } + const lastCodeUnit = chunk.charCodeAt(chunk.length - 1); + if (0xD800 <= lastCodeUnit && lastCodeUnit <= 0xDBFF) { + this.#pendingHighSurrogate = chunk.slice(-1); + chunk = chunk.slice(0, -1); + } else { + this.#pendingHighSurrogate = null; + } + if (chunk) { + controller.enqueue(core.encode(chunk)); + } + return Promise.resolve(); + } catch (err) { + return Promise.reject(err); + } + }, + flush: (controller) => { + try { + if (this.#pendingHighSurrogate !== null) { + controller.enqueue(new Uint8Array([0xEF, 0xBF, 0xBD])); + } + return Promise.resolve(); + } catch (err) { + return Promise.reject(err); + } + }, + }); + this[webidl.brand] = webidl.brand; + } + + /** @returns {string} */ + get encoding() { + webidl.assertBranded(this, TextEncoderStream); + return "utf-8"; + } + + /** @returns {ReadableStream} */ + get readable() { + webidl.assertBranded(this, TextEncoderStream); + return this.#transform.readable; + } + + /** @returns {WritableStream} */ + get writable() { + webidl.assertBranded(this, TextEncoderStream); + return this.#transform.writable; + } + + get [Symbol.toStringTag]() { + return "TextEncoderStream"; + } + } + + Object.defineProperty(TextEncoderStream.prototype, "encoding", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextEncoderStream.prototype, "readable", { + enumerable: true, + configurable: true, + }); + Object.defineProperty(TextEncoderStream.prototype, "writable", { + enumerable: true, + configurable: true, + }); + webidl.converters.TextDecoderOptions = webidl.createDictionaryConverter( "TextDecoderOptions", [ @@ -259,6 +451,8 @@ window.__bootstrap.encoding = { TextEncoder, TextDecoder, + TextEncoderStream, + TextDecoderStream, decode, }; })(this); diff --git a/extensions/web/lib.deno_web.d.ts b/extensions/web/lib.deno_web.d.ts index e915345672..0c36733515 100644 --- a/extensions/web/lib.deno_web.d.ts +++ b/extensions/web/lib.deno_web.d.ts @@ -195,8 +195,8 @@ declare class TextDecoder { readonly fatal: boolean; /** Returns `true` if ignore BOM flag is set, and `false` otherwise. */ readonly ignoreBOM = false; - /** Returns the result of running encoding's decoder. */ + /** Returns the result of running encoding's decoder. */ decode(input?: BufferSource, options?: TextDecodeOptions): string; } @@ -207,12 +207,33 @@ declare interface TextEncoderEncodeIntoResult { declare class TextEncoder { /** Returns "utf-8". */ - readonly encoding = "utf-8"; + readonly encoding: "utf-8"; /** Returns the result of running UTF-8's encoder. */ encode(input?: string): Uint8Array; encodeInto(input: string, dest: Uint8Array): TextEncoderEncodeIntoResult; } +declare class TextDecoderStream { + /** Returns encoding's name, lowercased. */ + readonly encoding: string; + /** Returns `true` if error mode is "fatal", and `false` otherwise. */ + readonly fatal: boolean; + /** Returns `true` if ignore BOM flag is set, and `false` otherwise. */ + readonly ignoreBOM = false; + constructor(label?: string, options?: TextDecoderOptions); + readonly readable: ReadableStream; + readonly writable: WritableStream; + readonly [Symbol.toStringTag]: string; +} + +declare class TextEncoderStream { + /** Returns "utf-8". */ + readonly encoding: "utf-8"; + readonly readable: ReadableStream; + readonly writable: WritableStream; + readonly [Symbol.toStringTag]: string; +} + /** A controller object that allows you to abort one or more DOM requests as and * when desired. */ declare class AbortController { diff --git a/runtime/js/99_main.js b/runtime/js/99_main.js index 7ec422c1f8..6c79468d1e 100644 --- a/runtime/js/99_main.js +++ b/runtime/js/99_main.js @@ -287,6 +287,8 @@ delete Object.prototype.__proto__; Response: util.nonEnumerable(fetch.Response), TextDecoder: util.nonEnumerable(encoding.TextDecoder), TextEncoder: util.nonEnumerable(encoding.TextEncoder), + TextDecoderStream: util.nonEnumerable(encoding.TextDecoderStream), + TextEncoderStream: util.nonEnumerable(encoding.TextEncoderStream), TransformStream: util.nonEnumerable(streams.TransformStream), URL: util.nonEnumerable(url.URL), URLSearchParams: util.nonEnumerable(url.URLSearchParams), diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json index a941864e2b..d87017c9eb 100644 --- a/tools/wpt/expectation.json +++ b/tools/wpt/expectation.json @@ -186,24 +186,7 @@ "encodeInto.any.html": [ "encodeInto() and a detached output buffer" ], - "idlharness.any.html": [ - "TextDecoderStream interface: existence and properties of interface object", - "TextDecoderStream interface object length", - "TextDecoderStream interface object name", - "TextDecoderStream interface: existence and properties of interface prototype object", - "TextDecoderStream interface: existence and properties of interface prototype object's \"constructor\" property", - "TextDecoderStream interface: existence and properties of interface prototype object's @@unscopables property", - "TextDecoderStream interface: attribute encoding", - "TextDecoderStream interface: attribute fatal", - "TextDecoderStream interface: attribute ignoreBOM", - "TextEncoderStream interface: existence and properties of interface object", - "TextEncoderStream interface object length", - "TextEncoderStream interface object name", - "TextEncoderStream interface: existence and properties of interface prototype object", - "TextEncoderStream interface: existence and properties of interface prototype object's \"constructor\" property", - "TextEncoderStream interface: existence and properties of interface prototype object's @@unscopables property", - "TextEncoderStream interface: attribute encoding" - ], + "idlharness.any.html": true, "iso-2022-jp-decoder.any.html": true, "legacy-mb-schinese": { "gb18030": { @@ -215,18 +198,20 @@ }, "replacement-encodings.any.html": false, "streams": { - "backpressure.any.html": false, - "decode-attributes.any.html": false, - "decode-bad-chunks.any.html": false, - "decode-ignore-bom.any.html": false, - "decode-incomplete-input.any.html": false, - "decode-non-utf8.any.html": false, - "decode-split-character.any.html": false, - "decode-utf8.any.html": false, - "encode-bad-chunks.any.html": false, - "encode-utf8.any.html": false, - "readable-writable-properties.any.html": false, - "realms.window.html": false + "backpressure.any.html": true, + "decode-attributes.any.html": true, + "decode-bad-chunks.any.html": true, + "decode-ignore-bom.any.html": true, + "decode-incomplete-input.any.html": true, + "decode-non-utf8.any.html": true, + "decode-split-character.any.html": true, + "decode-utf8.any.html": [ + "decoding a transferred Uint8Array chunk should give no output", + "decoding a transferred ArrayBuffer chunk should give no output" + ], + "encode-bad-chunks.any.html": true, + "encode-utf8.any.html": true, + "readable-writable-properties.any.html": true }, "textdecoder-arguments.any.html": true, "textdecoder-byte-order-marks.any.html": true,