From 54467015e0017a7a5fe9dd56eb21458325e2f3e9 Mon Sep 17 00:00:00 2001
From: Divy Srivastava <dj.srivastava23@gmail.com>
Date: Thu, 3 Oct 2024 18:31:36 +0530
Subject: [PATCH] chore: show expectation diff for wpt tests (#26014)

Closes https://github.com/denoland/deno/issues/26012

```
========================================

failures:

        "/WebCryptoAPI/wrapKey_unwrapKey/wrapKey_unwrapKey.https.any.html - Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CTR"

final result: failed. 1 passed; 1 failed; 0 expected failure; total 2 (15646ms)

diff --git a/Users/divy/gh/deno/tests/wpt/runner/expectation.json b/var/folders/ll/ltqsx4nx72v5_r7rlsl36pgm0000gn/T/375d79f1257b05cd
index fb2063935..4449c5d15 100644
--- a/Users/divy/gh/deno/tests/wpt/runner/expectation.json
+++ b/var/folders/ll/ltqsx4nx72v5_r7rlsl36pgm0000gn/T/375d79f1257b05cd
@@ -1531,6 +1531,7 @@
     },
     "wrapKey_unwrapKey": {
       "wrapKey_unwrapKey.https.any.html": [
+        "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CTR",
         "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-CBC",
         "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-GCM",
         "Can wrap and unwrap X25519 private key keys as non-extractable using pkcs8 and AES-KW",
```
---
 tests/wpt/runner/utils.ts | 21 ++++++++++++++++++---
 tests/wpt/wpt.ts          | 32 +++++++++++++++++++++++++-------
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/tests/wpt/runner/utils.ts b/tests/wpt/runner/utils.ts
index 1674419cd1..140c388ec2 100644
--- a/tests/wpt/runner/utils.ts
+++ b/tests/wpt/runner/utils.ts
@@ -76,7 +76,10 @@ export function getManifest(): Manifest {
 
 /// WPT TEST EXPECTATIONS
 
-const EXPECTATION_PATH = join(ROOT_PATH, "./tests/wpt/runner/expectation.json");
+export const EXPECTATION_PATH = join(
+  ROOT_PATH,
+  "./tests/wpt/runner/expectation.json",
+);
 
 export interface Expectation {
   [key: string]: Expectation | boolean | string[] | { ignore: boolean };
@@ -87,9 +90,12 @@ export function getExpectation(): Expectation {
   return JSON.parse(expectationText);
 }
 
-export function saveExpectation(expectation: Expectation) {
+export function saveExpectation(
+  expectation: Expectation,
+  path: string = EXPECTATION_PATH,
+) {
   Deno.writeTextFileSync(
-    EXPECTATION_PATH,
+    path,
     JSON.stringify(expectation, undefined, "  ") + "\n",
   );
 }
@@ -134,6 +140,15 @@ export function runPy<T extends Omit<Deno.CommandOptions, "cwd">>(
   }).spawn();
 }
 
+export async function runGitDiff(args: string[]): string {
+  await new Deno.Command("git", {
+    args: ["diff", ...args],
+    stdout: "inherit",
+    stderr: "inherit",
+    cwd: ROOT_PATH,
+  }).output();
+}
+
 export async function checkPy3Available() {
   const { success, stdout } = await runPy(["--version"], {
     stdout: "piped",
diff --git a/tests/wpt/wpt.ts b/tests/wpt/wpt.ts
index c42ff51e6e..b13a10cf4c 100755
--- a/tests/wpt/wpt.ts
+++ b/tests/wpt/wpt.ts
@@ -18,6 +18,7 @@ import {
   checkPy3Available,
   escapeLoneSurrogates,
   Expectation,
+  EXPECTATION_PATH,
   generateRunInfo,
   getExpectation,
   getExpectFailForCase,
@@ -30,6 +31,7 @@ import {
   noIgnore,
   quiet,
   rest,
+  runGitDiff,
   runPy,
   updateManifest,
   wptreport,
@@ -256,7 +258,16 @@ async function run() {
     await Deno.writeTextFile(wptreport, JSON.stringify(report) + "\n");
   }
 
+  const newExpectations = newExpectation(results);
+  const tmp = Deno.makeTempFileSync();
+  saveExpectation(newExpectations, tmp);
+
   const code = reportFinal(results, endTime - startTime);
+
+  // Run git diff to see what changed
+  await runGitDiff([EXPECTATION_PATH, tmp]);
+  Deno.removeSync(tmp);
+
   Deno.exit(code);
 }
 
@@ -390,6 +401,19 @@ async function update() {
     await Deno.writeTextFile(json, JSON.stringify(results) + "\n");
   }
 
+  const newExpectations = newExpectation(results);
+  saveExpectation(newExpectations);
+
+  reportFinal(results, endTime - startTime);
+
+  console.log(blue("Updated expectation.json to match reality."));
+
+  Deno.exit(0);
+}
+
+function newExpectation(
+  results: { test: TestToRun; result: TestResult }[],
+): Expectation {
   const resultTests: Record<
     string,
     { passed: string[]; failed: string[]; testSucceeded: boolean }
@@ -431,13 +455,7 @@ async function update() {
     );
   }
 
-  saveExpectation(currentExpectation);
-
-  reportFinal(results, endTime - startTime);
-
-  console.log(blue("Updated expectation.json to match reality."));
-
-  Deno.exit(0);
+  return currentExpectation;
 }
 
 function insertExpectation(