From ba40347a35d10cffeae83ee95f69f7bc281096dd Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Tue, 6 Aug 2024 00:13:02 -0700 Subject: [PATCH] feat(fetch): accept async iterables for body (#24623) Implements https://github.com/whatwg/webidl/pull/1397 Fixes #21454 Closes #24849 --- ext/fetch/22_body.js | 13 +++ ext/fetch/lib.deno_fetch.d.ts | 1 + ext/web/06_streams.js | 63 ++++---------- ext/webidl/00_webidl.js | 126 +++++++++++++++++++++++++++ ext/webidl/internal.d.ts | 21 +++++ tests/integration/node_unit_tests.rs | 1 + tests/unit/streams_test.ts | 15 +++- tests/unit_node/fetch_test.ts | 18 ++++ tests/wpt/runner/expectation.json | 10 ++- 9 files changed, 217 insertions(+), 51 deletions(-) create mode 100644 tests/unit_node/fetch_test.ts diff --git a/ext/fetch/22_body.js b/ext/fetch/22_body.js index 82f41411d8..ae5aef8acb 100644 --- a/ext/fetch/22_body.js +++ b/ext/fetch/22_body.js @@ -458,6 +458,8 @@ function extractBody(object) { if (object.locked || isReadableStreamDisturbed(object)) { throw new TypeError("ReadableStream is locked or disturbed"); } + } else if (object[webidl.AsyncIterable] === webidl.AsyncIterable) { + stream = ReadableStream.from(object.open()); } if (typeof source === "string") { // WARNING: this deviates from spec (expects length to be set) @@ -475,6 +477,9 @@ function extractBody(object) { return { body, contentType }; } +webidl.converters["async iterable"] = webidl + .createAsyncIterableConverter(webidl.converters.Uint8Array); + webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { // Union for (ReadableStream or Blob or ArrayBufferView or ArrayBuffer or FormData or URLSearchParams or USVString) if (ObjectPrototypeIsPrototypeOf(ReadableStreamPrototype, V)) { @@ -493,6 +498,14 @@ webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => { if (ArrayBufferIsView(V)) { return webidl.converters["ArrayBufferView"](V, prefix, context, opts); } + if (webidl.isAsyncIterator(V)) { + return webidl.converters["async iterable"]( + V, + prefix, + context, + opts, + ); + } } // BodyInit conversion is passed to extractBody(), which calls core.encode(). // core.encode() will UTF-8 encode strings with replacement, being equivalent to the USV normalization. diff --git a/ext/fetch/lib.deno_fetch.d.ts b/ext/fetch/lib.deno_fetch.d.ts index c27313903d..3bf608cdb4 100644 --- a/ext/fetch/lib.deno_fetch.d.ts +++ b/ext/fetch/lib.deno_fetch.d.ts @@ -163,6 +163,7 @@ declare type BodyInit = | FormData | URLSearchParams | ReadableStream + | AsyncIterable | string; /** @category Fetch */ declare type RequestDestination = diff --git a/ext/web/06_streams.js b/ext/web/06_streams.js index d3de243637..ea3a3110f6 100644 --- a/ext/web/06_streams.js +++ b/ext/web/06_streams.js @@ -70,7 +70,6 @@ const { String, Symbol, SymbolAsyncIterator, - SymbolIterator, SymbolFor, TypeError, TypedArrayPrototypeGetBuffer, @@ -5083,34 +5082,6 @@ function initializeCountSizeFunction(globalObject) { WeakMapPrototypeSet(countSizeFunctionWeakMap, globalObject, size); } -// Ref: https://tc39.es/ecma262/#sec-getiterator -function getAsyncOrSyncIterator(obj) { - let iterator; - if (obj[SymbolAsyncIterator] != null) { - iterator = obj[SymbolAsyncIterator](); - if (!isObject(iterator)) { - throw new TypeError( - "[Symbol.asyncIterator] returned a non-object value", - ); - } - } else if (obj[SymbolIterator] != null) { - iterator = obj[SymbolIterator](); - if (!isObject(iterator)) { - throw new TypeError("[Symbol.iterator] returned a non-object value"); - } - } else { - throw new TypeError("No iterator found"); - } - if (typeof iterator.next !== "function") { - throw new TypeError("iterator.next is not a function"); - } - return iterator; -} - -function isObject(x) { - return (typeof x === "object" && x != null) || typeof x === "function"; -} - const _resourceBacking = Symbol("[[resourceBacking]]"); // This distinction exists to prevent unrefable streams being used in // regular fast streams that are unaware of refability @@ -5196,21 +5167,22 @@ class ReadableStream { } static from(asyncIterable) { + const prefix = "Failed to execute 'ReadableStream.from'"; webidl.requiredArguments( arguments.length, 1, - "Failed to execute 'ReadableStream.from'", + prefix, ); - asyncIterable = webidl.converters.any(asyncIterable); - - const iterator = getAsyncOrSyncIterator(asyncIterable); + asyncIterable = webidl.converters["async iterable"]( + asyncIterable, + prefix, + "Argument 1", + ); + const iter = asyncIterable.open(); const stream = createReadableStream(noop, async () => { // deno-lint-ignore prefer-primordials - const res = await iterator.next(); - if (!isObject(res)) { - throw new TypeError("iterator.next value is not an object"); - } + const res = await iter.next(); if (res.done) { readableStreamDefaultControllerClose(stream[_controller]); } else { @@ -5220,17 +5192,8 @@ class ReadableStream { ); } }, async (reason) => { - if (iterator.return == null) { - return undefined; - } else { - // deno-lint-ignore prefer-primordials - const res = await iterator.return(reason); - if (!isObject(res)) { - throw new TypeError("iterator.return value is not an object"); - } else { - return undefined; - } - } + // deno-lint-ignore prefer-primordials + await iter.return(reason); }, 0); return stream; } @@ -6890,6 +6853,10 @@ webidl.converters.StreamPipeOptions = webidl { key: "signal", converter: webidl.converters.AbortSignal }, ]); +webidl.converters["async iterable"] = webidl.createAsyncIterableConverter( + webidl.converters.any, +); + internals.resourceForReadableStream = resourceForReadableStream; export { diff --git a/ext/webidl/00_webidl.js b/ext/webidl/00_webidl.js index 9ea2200f33..7440e47e7b 100644 --- a/ext/webidl/00_webidl.js +++ b/ext/webidl/00_webidl.js @@ -26,6 +26,7 @@ const { Float32Array, Float64Array, FunctionPrototypeBind, + FunctionPrototypeCall, Int16Array, Int32Array, Int8Array, @@ -77,6 +78,7 @@ const { StringPrototypeToWellFormed, Symbol, SymbolIterator, + SymbolAsyncIterator, SymbolToStringTag, TypedArrayPrototypeGetBuffer, TypedArrayPrototypeGetSymbolToStringTag, @@ -919,6 +921,127 @@ function createSequenceConverter(converter) { }; } +function isAsyncIterator(obj) { + if (obj[SymbolAsyncIterator] === undefined) { + if (obj[SymbolIterator] === undefined) { + return false; + } + } + + return true; +} + +const AsyncIterable = Symbol("[[asyncIterable]]"); + +function createAsyncIterableConverter(converter) { + return function ( + V, + prefix = undefined, + context = undefined, + opts = { __proto__: null }, + ) { + if (type(V) !== "Object") { + throw makeException( + TypeError, + "can not be converted to async iterable.", + prefix, + context, + ); + } + + let isAsync = true; + let method = V[SymbolAsyncIterator]; + if (method === undefined) { + method = V[SymbolIterator]; + + if (method === undefined) { + throw makeException( + TypeError, + "is not iterable.", + prefix, + context, + ); + } + + isAsync = false; + } + + return { + value: V, + [AsyncIterable]: AsyncIterable, + open(context) { + const iter = FunctionPrototypeCall(method, V); + if (type(iter) !== "Object") { + throw new TypeError( + `${context} could not be iterated because iterator method did not return object, but ${ + type(iter) + }.`, + ); + } + + let asyncIterator = iter; + + if (!isAsync) { + asyncIterator = { + // deno-lint-ignore require-await + async next() { + // deno-lint-ignore prefer-primordials + return iter.next(); + }, + }; + } + + return { + async next() { + // deno-lint-ignore prefer-primordials + const iterResult = await asyncIterator.next(); + if (type(iterResult) !== "Object") { + throw TypeError( + `${context} failed to iterate next value because the next() method did not return an object, but ${ + type(iterResult) + }.`, + ); + } + + if (iterResult.done) { + return { done: true }; + } + + const iterValue = converter( + iterResult.value, + `${context} failed to iterate next value`, + `The value returned from the next() method`, + opts, + ); + + return { done: false, value: iterValue }; + }, + async return(reason) { + if (asyncIterator.return === undefined) { + return undefined; + } + + // deno-lint-ignore prefer-primordials + const returnPromiseResult = await asyncIterator.return(reason); + if (type(returnPromiseResult) !== "Object") { + throw TypeError( + `${context} failed to close iterator because the return() method did not return an object, but ${ + type(returnPromiseResult) + }.`, + ); + } + + return undefined; + }, + [SymbolAsyncIterator]() { + return this; + }, + }; + }, + }; + }; +} + function createRecordConverter(keyConverter, valueConverter) { return (V, prefix, context, opts) => { if (type(V) !== "Object") { @@ -1287,9 +1410,11 @@ function setlike(obj, objPrototype, readonly) { export { assertBranded, + AsyncIterable, brand, configureInterface, converters, + createAsyncIterableConverter, createBranded, createDictionaryConverter, createEnumConverter, @@ -1300,6 +1425,7 @@ export { createSequenceConverter, illegalConstructor, invokeCallbackFunction, + isAsyncIterator, makeException, mixinPairIterable, requiredArguments, diff --git a/ext/webidl/internal.d.ts b/ext/webidl/internal.d.ts index 1ce45463ec..d9266f5f54 100644 --- a/ext/webidl/internal.d.ts +++ b/ext/webidl/internal.d.ts @@ -438,6 +438,27 @@ declare module "ext:deno_webidl/00_webidl.js" { opts?: any, ) => T[]; + /** + * Create a converter that converts an async iterable of the inner type. + */ + function createAsyncIterableConverter( + converter: ( + v: V, + prefix?: string, + context?: string, + opts?: any, + ) => T, + ): ( + v: any, + prefix?: string, + context?: string, + opts?: any, + ) => ConvertedAsyncIterable; + + interface ConvertedAsyncIterable extends AsyncIterableIterator { + value: V; + } + /** * Create a converter that converts a Promise of the inner type. */ diff --git a/tests/integration/node_unit_tests.rs b/tests/integration/node_unit_tests.rs index 4f0b613788..e87f1cd2f8 100644 --- a/tests/integration/node_unit_tests.rs +++ b/tests/integration/node_unit_tests.rs @@ -71,6 +71,7 @@ util::unit_test_factory!( dgram_test, domain_test, fs_test, + fetch_test, http_test, http2_test, _randomBytes_test = internal / _randomBytes_test, diff --git a/tests/unit/streams_test.ts b/tests/unit/streams_test.ts index 80b45e6024..c0adbda07c 100644 --- a/tests/unit/streams_test.ts +++ b/tests/unit/streams_test.ts @@ -1,5 +1,10 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assertEquals, assertRejects, fail } from "./test_util.ts"; +import { + assertEquals, + assertRejects, + assertThrows, + fail, +} from "./test_util.ts"; const { core, @@ -533,3 +538,11 @@ Deno.test(async function decompressionStreamInvalidGzipStillReported() { "corrupt gzip stream does not have a matching checksum", ); }); + +Deno.test(function readableStreamFromWithStringThrows() { + assertThrows( + () => ReadableStream.from("string"), + TypeError, + "Failed to execute 'ReadableStream.from': Argument 1 can not be converted to async iterable.", + ); +}); diff --git a/tests/unit_node/fetch_test.ts b/tests/unit_node/fetch_test.ts new file mode 100644 index 0000000000..399d6052a5 --- /dev/null +++ b/tests/unit_node/fetch_test.ts @@ -0,0 +1,18 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +import { assertEquals } from "@std/assert"; +import { createReadStream } from "node:fs"; + +Deno.test("fetch node stream", async () => { + const file = createReadStream("tests/testdata/assets/fixture.json"); + + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: file, + }); + + assertEquals( + await response.text(), + await Deno.readTextFile("tests/testdata/assets/fixture.json"), + ); +}); diff --git a/tests/wpt/runner/expectation.json b/tests/wpt/runner/expectation.json index 70e32f92f7..f59806feae 100644 --- a/tests/wpt/runner/expectation.json +++ b/tests/wpt/runner/expectation.json @@ -4007,8 +4007,14 @@ "owning-type-message-port.any.worker.html": false, "owning-type.any.html": false, "owning-type.any.worker.html": false, - "from.any.html": true, - "from.any.worker.html": true + "from.any.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ], + "from.any.worker.html": [ + "ReadableStream.from ignores a null @@asyncIterator", + "ReadableStream.from accepts a string" + ] }, "transform-streams": { "backpressure.any.html": true,