From 706b75d7421a33a54f51ceafc0747be4a646c707 Mon Sep 17 00:00:00 2001 From: Luca Casonato Date: Sat, 5 Jun 2021 19:29:41 +0200 Subject: [PATCH] tests: fix wpt runner expected test file failures (#10856) --- tools/wpt.ts | 32 ++++++++++++++++--- tools/wpt/expectation.json | 57 +++++++++++++++++++++++++--------- tools/wpt/testharnessreport.js | 9 ------ 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/tools/wpt.ts b/tools/wpt.ts index 203b198d17..e6f08dd34b 100755 --- a/tools/wpt.ts +++ b/tools/wpt.ts @@ -349,8 +349,14 @@ function reportFinal( let finalExpectedFailedAndFailedCount = 0; const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = []; const finalExpectedFailedButPassedFiles: string[] = []; + const finalFailedFiles: string[] = []; for (const { test, result } of results) { - const { failed, failedCount, expectedFailedButPassed } = analyzeTestResult( + const { + failed, + failedCount, + expectedFailedButPassed, + expectedFailedAndFailedCount, + } = analyzeTestResult( result, test.expectation, ); @@ -359,7 +365,7 @@ function reportFinal( finalExpectedFailedAndFailedCount += 1; } else { finalFailedCount += 1; - finalExpectedFailedButPassedFiles.push(test.path); + finalFailedFiles.push(test.path); } } else if (failedCount > 0) { finalFailedCount += 1; @@ -369,6 +375,11 @@ function reportFinal( for (const case_ of expectedFailedButPassed) { finalExpectedFailedButPassedTests.push([test.path, case_]); } + } else if ( + test.expectation === false && + expectedFailedAndFailedCount != result.cases.length + ) { + finalExpectedFailedButPassedFiles.push(test.path); } } const finalPassedCount = finalTotalCount - finalFailedCount; @@ -383,6 +394,14 @@ function reportFinal( ` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`, ); } + if (finalFailedFiles.length > 0) { + console.log(`\nfile failures:\n`); + } + for (const result of finalFailedFiles) { + console.log( + ` ${JSON.stringify(result)}`, + ); + } if (finalExpectedFailedButPassedTests.length > 0) { console.log(`\nexpected test failures that passed:\n`); } @@ -398,13 +417,16 @@ function reportFinal( console.log(` ${JSON.stringify(result)}`); } + const failed = (finalFailedCount > 0) || + (finalExpectedFailedButPassedFiles.length > 0); + console.log( `\nfinal result: ${ - finalFailedCount > 0 ? red("failed") : green("ok") + failed ? red("failed") : green("ok") }. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount}\n`, ); - return finalFailedCount > 0 ? 1 : 0; + return failed ? 1 : 0; } function analyzeTestResult( @@ -475,7 +497,7 @@ function reportVariation(result: TestResult, expectation: boolean | string[]) { console.log(`\n${result.name}\n${result.message}\n${result.stack}`); } - if (failed.length > 0) { + if (failedCount > 0) { console.log(`\nfailures:\n`); } for (const result of failed) { diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json index a8f9b89a95..dd5f9d3b6b 100644 --- a/tools/wpt/expectation.json +++ b/tools/wpt/expectation.json @@ -174,7 +174,9 @@ "event.any.html": true }, "events": { - "AddEventListenerOptions-signal.any.html": true, + "AddEventListenerOptions-signal.any.html": [ + "Passing null as the signal should throw" + ], "Event-isTrusted.any.html": true, "EventTarget-constructible.any.html": true, "Event-constructors.any.html": [ @@ -535,6 +537,7 @@ "getReader({mode: \"byob\"}) throws on non-bytes streams", "ReadableStream with byte source can be constructed with no errors", "getReader({mode}) must perform ToString()", + "ReadableStream with byte source: autoAllocateChunkSize cannot be 0", "ReadableStreamBYOBReader can be constructed directly", "ReadableStreamBYOBReader constructor requires a ReadableStream argument", "ReadableStreamBYOBReader constructor requires an unlocked ReadableStream", @@ -589,7 +592,8 @@ "ReadableStream with byte source: respondWithNewView() with a zero-length view (in the closed state)", "ReadableStream with byte source: respondWithNewView() with a transferred non-zero-length view (in the readable state)", "ReadableStream with byte source: respondWithNewView() with a transferred zero-length view (in the closed state)" - ] + ], + "non-transferable-buffers.any.html": false }, "readable-streams": { "async-iterator.any.html": [ @@ -648,6 +652,7 @@ "writable-streams": { "aborting.any.html": false, "bad-strategies.any.html": [ + "reject any non-function value for strategy.size", "Writable stream: invalid size beats invalid highWaterMark" ], "bad-underlying-sinks.any.html": true, @@ -655,6 +660,8 @@ "close.any.html": false, "constructor.any.html": [ "underlyingSink argument should be converted after queuingStrategy argument", + "WritableStreamDefaultController constructor should throw", + "WritableStreamDefaultController constructor should throw when passed an initialised WritableStream", "WritableStreamDefaultWriter should throw unless passed a WritableStream" ], "count-queuing-strategy.any.html": true, @@ -684,7 +691,9 @@ "Performance interface: operation mark(DOMString, optional PerformanceMarkOptions)", "Performance interface: operation clearMarks(optional DOMString)", "Performance interface: operation measure(DOMString, optional (DOMString or PerformanceMeasureOptions), optional DOMString)", - "Performance interface: operation clearMeasures(optional DOMString)" + "Performance interface: operation clearMeasures(optional DOMString)", + "Performance interface: calling mark(DOMString, optional PerformanceMarkOptions) on performance with too few arguments must throw TypeError", + "Performance interface: calling measure(DOMString, optional (DOMString or PerformanceMeasureOptions), optional DOMString) on performance with too few arguments must throw TypeError" ], "mark-entry-constructor.any.html": true, "mark-errors.any.html": true, @@ -743,10 +752,7 @@ "toString.any.html": true, "type.tentative.any.html": false, "constructor-shared.tentative.any.html": true, - "constructor-types.tentative.any.html": [ - "Zero minimum", - "Non-zero minimum" - ] + "constructor-types.tentative.any.html": false }, "module": { "constructor.any.html": true, @@ -766,12 +772,10 @@ "toString.any.html": true, "constructor-reftypes.tentative.any.html": [ "initialize externref table with default value", - "initialize anyfunc table with default value" - ], - "constructor-types.tentative.any.html": [ - "Zero minimum", - "Non-zero minimum" + "initialize anyfunc table with default value", + "initialize anyfunc table with a bad default value" ], + "constructor-types.tentative.any.html": false, "grow-reftypes.tentative.any.html": false, "set-reftypes.tentative.any.html": false, "type.tentative.any.html": false @@ -835,7 +839,6 @@ "URL interface: attribute searchParams", "URL interface: attribute hash", "URL interface: operation toJSON()", - "URL interface: legacy window alias", "Stringification of new URL(\"http://foo\")", "URLSearchParams interface: operation append(USVString, USVString)", "URLSearchParams interface: operation delete(USVString)", @@ -849,6 +852,12 @@ "Stringification of new URLSearchParams(\"hi=there&thank=you\")" ], "url-constructor.any.html": [ + "Parsing: against ", + "Parsing: against ", + "Parsing: against ", + "Parsing: against ", + "Parsing: against ", + "Parsing: against ", "Parsing: against ", "Parsing: against ", "Parsing: against ", @@ -933,13 +942,30 @@ "urlsearchparams-has.any.html": true, "urlsearchparams-set.any.html": true, "urlsearchparams-sort.any.html": true, - "urlsearchparams-stringifier.any.html": true + "urlsearchparams-stringifier.any.html": true, + "url-setters.any.html": [ + "URL: Setting .hostname = 'example.com:8080' : delimiter invalidates entire value", + "URL: Setting .hostname = 'example.com:' : delimiter invalidates entire value", + "URL: Setting .hostname = 'h' Drop /. from path", + "URL: Setting .hostname = ''", + "URL: Setting .pathname = '\\\\' File URLs and (back)slashes", + "URL: Setting .pathname = '//\\/' File URLs and (back)slashes", + "URL: Setting .pathname = '//monkey/..//' File URLs and (back)slashes", + "URL: Setting .pathname = '/.//p' Serialize /. in path", + "URL: Setting .pathname = '/..//p'", + "URL: Setting .pathname = '//p'", + "URL: Setting .pathname = 'p' Drop /. from path" + ] }, "fetch": { "api": { "request": { "request-init-002.any.html": true, - "request-init-stream.any.html": true, + "request-init-stream.any.html": [ + "Constructing a Request with a Request on which body.getReader() is called", + "Constructing a Request with a Request on which body.getReader().read() is called", + "Constructing a Request with a Request on which read() and releaseLock() are called" + ], "request-consume-empty.any.html": [ "Consume empty FormData request body as text" ], @@ -1123,6 +1149,7 @@ "Response interface: operation redirect(USVString, optional unsigned short)", "Response interface: attribute body", "Response interface: attribute bodyUsed", + "Response interface: calling redirect(USVString, optional unsigned short) on new Response() with too few arguments must throw TypeError", "Window interface: operation fetch(RequestInfo, optional RequestInit)" ] }, diff --git a/tools/wpt/testharnessreport.js b/tools/wpt/testharnessreport.js index b1852a4886..d3e7833767 100644 --- a/tools/wpt/testharnessreport.js +++ b/tools/wpt/testharnessreport.js @@ -13,12 +13,3 @@ window.add_result_callback(({ message, name, stack, status }) => { window.add_completion_callback((_tests, _harnessStatus) => { Deno.exit(0); }); - -globalThis.document = { - // document.body shim for FileAPI/file/File-constructor.any.html test - body: { - toString() { - return "[object HTMLBodyElement]"; - }, - }, -};