chore: add script to check remaining node compat cases (#27122)

tests/node_compat/common.ts

@@ -2,6 +2,9 @@
 import { partition } from "@std/collections/partition";
 import { join } from "@std/path";
 import * as JSONC from "@std/jsonc";
+import { walk } from "@std/fs/walk";
+import { relative } from "@std/path/posix/relative";
+
 /**
  * The test suite matches the folders inside the `test` folder inside the
  * node repo
@@ -61,3 +64,130 @@ export function partitionParallelTestPaths(
   const partitions = partition(testPaths, (p) => !!p.match(PARALLEL_PATTERN));
   return { parallel: partitions[0], sequential: partitions[1] };
 }
+
+export const NODE_IGNORED_TEST_DIRS = [
+  "addons",
+  "async-hooks",
+  "cctest",
+  "common",
+  "doctool",
+  "embedding",
+  "fixtures",
+  "fuzzers",
+  "js-native-api",
+  "node-api",
+  "overlapped-checker",
+  "report",
+  "testpy",
+  "tick-processor",
+  "tools",
+  "v8-updates",
+  "wasi",
+  "wpt",
+];
+
+export const VENDORED_NODE_TEST = new URL(
+  "./runner/suite/test/",
+  import.meta.url,
+);
+export const NODE_COMPAT_TEST_DEST_URL = new URL(
+  "./test/",
+  import.meta.url,
+);
+
+export async function getNodeTests(): Promise<string[]> {
+  const paths: string[] = [];
+  const rootPath = VENDORED_NODE_TEST.href.slice(7);
+  for await (
+    const item of walk(VENDORED_NODE_TEST, { exts: [".js"] })
+  ) {
+    const path = relative(rootPath, item.path);
+    if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) {
+      paths.push(path);
+    }
+  }
+
+  return paths.sort();
+}
+
+export async function getDenoTests() {
+  const paths: string[] = [];
+  const rootPath = NODE_COMPAT_TEST_DEST_URL.href.slice(7);
+  for await (
+    const item of walk(NODE_COMPAT_TEST_DEST_URL, { exts: [".js"] })
+  ) {
+    const path = relative(rootPath, item.path);
+    paths.push(path);
+  }
+
+  return paths.sort();
+}
+
+let testSerialId = 0;
+const cwd = new URL(".", import.meta.url);
+
+export async function runNodeCompatTestCase(
+  testCase: string,
+  signal?: AbortSignal,
+) {
+  const v8Flags = ["--stack-size=4000"];
+  const testSource = await Deno.readTextFile(testCase);
+  const envVars: Record<string, string> = {};
+  const knownGlobals: string[] = [];
+  parseFlags(testSource).forEach((flag) => {
+    switch (flag) {
+      case "--expose_externalize_string":
+        v8Flags.push("--expose-externalize-string");
+        knownGlobals.push("createExternalizableString");
+        break;
+      case "--expose-gc":
+        v8Flags.push("--expose-gc");
+        knownGlobals.push("gc");
+        break;
+      default:
+        break;
+    }
+  });
+  if (knownGlobals.length > 0) {
+    envVars["NODE_TEST_KNOWN_GLOBALS"] = knownGlobals.join(",");
+  }
+  // TODO(nathanwhit): once we match node's behavior on executing
+  // `node:test` tests when we run a file, we can remove this
+  const usesNodeTest = testSource.includes("node:test");
+  const args = [
+    usesNodeTest ? "test" : "run",
+    "-A",
+    "--quiet",
+    "--unstable-unsafe-proto",
+    "--unstable-bare-node-builtins",
+    "--unstable-fs",
+    "--v8-flags=" + v8Flags.join(),
+  ];
+  if (usesNodeTest) {
+    // deno test typechecks by default + we want to pass script args
+    args.push("--no-check", "runner.ts", "--", testCase);
+  } else {
+    args.push("runner.ts", testCase);
+  }
+
+  // Pipe stdout in order to output each test result as Deno.test output
+  // That way the tests will respect the `--quiet` option when provided
+  return new Deno.Command(Deno.execPath(), {
+    args,
+    env: {
+      TEST_SERIAL_ID: String(testSerialId++),
+      ...envVars,
+    },
+    cwd,
+    stdout: "piped",
+    stderr: "piped",
+    signal,
+  }).spawn();
+}
+
+/** Parses the special "Flags:"" syntax in Node.js test files */
+function parseFlags(source: string): string[] {
+  const line = /^\/\/ Flags: (.+)$/um.exec(source);
+  if (line == null) return [];
+  return line[1].split(" ");
+}

tests/node_compat/runner/TODO.md

@@ -1,7 +1,7 @@
 <!-- deno-fmt-ignore-file -->
 # Remaining Node Tests
 
-1120 tests out of 3681 have been ported from Node 20.11.1 (30.43% ported, 69.68% remaining).
+1163 tests out of 3681 have been ported from Node 20.11.1 (31.59% ported, 68.92% remaining).
 
 NOTE: This file should not be manually edited. Please edit `tests/node_compat/config.json` and run `deno task setup` in `tests/node_compat/runner` dir instead.
 
@@ -223,18 +223,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-child-process-constructor.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-constructor.js)
 - [parallel/test-child-process-cwd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-cwd.js)
 - [parallel/test-child-process-destroy.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-destroy.js)
-- [parallel/test-child-process-detached.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-detached.js)
 - [parallel/test-child-process-disconnect.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-disconnect.js)
 - [parallel/test-child-process-env.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-env.js)
 - [parallel/test-child-process-exec-any-shells-windows.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-any-shells-windows.js)
-- [parallel/test-child-process-exec-encoding.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-encoding.js)
-- [parallel/test-child-process-exec-std-encoding.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-std-encoding.js)
 - [parallel/test-child-process-exec-timeout-expire.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-expire.js)
 - [parallel/test-child-process-exec-timeout-kill.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-kill.js)
-- [parallel/test-child-process-exec-timeout-not-expired.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exec-timeout-not-expired.js)
 - [parallel/test-child-process-execFile-promisified-abortController.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-execFile-promisified-abortController.js)
-- [parallel/test-child-process-execfile.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-execfile.js)
-- [parallel/test-child-process-exit-code.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-exit-code.js)
 - [parallel/test-child-process-fork-abort-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-abort-signal.js)
 - [parallel/test-child-process-fork-and-spawn.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-and-spawn.js)
 - [parallel/test-child-process-fork-args.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-args.js)
@@ -249,15 +243,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-child-process-fork-net-socket.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-net-socket.js)
 - [parallel/test-child-process-fork-net.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-net.js)
 - [parallel/test-child-process-fork-no-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-no-shell.js)
-- [parallel/test-child-process-fork-ref.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-ref.js)
-- [parallel/test-child-process-fork-ref2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-ref2.js)
 - [parallel/test-child-process-fork-stdio-string-variant.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-stdio-string-variant.js)
 - [parallel/test-child-process-fork-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-stdio.js)
 - [parallel/test-child-process-fork-timeout-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork-timeout-kill-signal.js)
 - [parallel/test-child-process-fork.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-fork.js)
 - [parallel/test-child-process-http-socket-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-http-socket-leak.js)
 - [parallel/test-child-process-internal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-internal.js)
-- [parallel/test-child-process-ipc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-ipc.js)
 - [parallel/test-child-process-no-deprecation.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-no-deprecation.js)
 - [parallel/test-child-process-pipe-dataflow.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-pipe-dataflow.js)
 - [parallel/test-child-process-promisified.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-promisified.js)
@@ -276,19 +267,15 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-child-process-spawn-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-shell.js)
 - [parallel/test-child-process-spawn-timeout-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-timeout-kill-signal.js)
 - [parallel/test-child-process-spawn-typeerror.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawn-typeerror.js)
-- [parallel/test-child-process-spawnsync-env.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-env.js)
 - [parallel/test-child-process-spawnsync-input.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-input.js)
 - [parallel/test-child-process-spawnsync-kill-signal.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-kill-signal.js)
 - [parallel/test-child-process-spawnsync-shell.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-shell.js)
 - [parallel/test-child-process-spawnsync-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-spawnsync-timeout.js)
 - [parallel/test-child-process-stdin.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdin.js)
 - [parallel/test-child-process-stdio-big-write-end.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-big-write-end.js)
-- [parallel/test-child-process-stdio-inherit.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-inherit.js)
 - [parallel/test-child-process-stdio-merge-stdouts-into-cat.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-merge-stdouts-into-cat.js)
 - [parallel/test-child-process-stdio-reuse-readable-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio-reuse-readable-stdio.js)
 - [parallel/test-child-process-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdio.js)
-- [parallel/test-child-process-stdout-flush-exit.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-flush-exit.js)
-- [parallel/test-child-process-stdout-flush.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-flush.js)
 - [parallel/test-child-process-stdout-ipc.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-stdout-ipc.js)
 - [parallel/test-child-process-uid-gid.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-uid-gid.js)
 - [parallel/test-child-process-validate-stdio.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-child-process-validate-stdio.js)
@@ -394,7 +381,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-common.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-common.js)
 - [parallel/test-console-clear.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-clear.js)
 - [parallel/test-console-count.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-count.js)
-- [parallel/test-console-instance.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-instance.js)
 - [parallel/test-console-issue-43095.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-issue-43095.js)
 - [parallel/test-console-methods.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-methods.js)
 - [parallel/test-console-stdio-setters.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-console-stdio-setters.js)
@@ -505,21 +491,17 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-dgram-cluster-close-in-listening.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-cluster-close-in-listening.js)
 - [parallel/test-dgram-create-socket-handle-fd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-create-socket-handle-fd.js)
 - [parallel/test-dgram-create-socket-handle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-create-socket-handle.js)
-- [parallel/test-dgram-custom-lookup.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-custom-lookup.js)
 - [parallel/test-dgram-deprecation-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-deprecation-error.js)
 - [parallel/test-dgram-exclusive-implicit-bind.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-exclusive-implicit-bind.js)
-- [parallel/test-dgram-ipv6only.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-ipv6only.js)
 - [parallel/test-dgram-membership.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-membership.js)
 - [parallel/test-dgram-multicast-loopback.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-loopback.js)
 - [parallel/test-dgram-multicast-set-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-set-interface.js)
 - [parallel/test-dgram-multicast-setTTL.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-multicast-setTTL.js)
 - [parallel/test-dgram-send-address-types.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-address-types.js)
-- [parallel/test-dgram-send-cb-quelches-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-cb-quelches-error.js)
 - [parallel/test-dgram-send-queue-info.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-send-queue-info.js)
 - [parallel/test-dgram-sendto.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-sendto.js)
 - [parallel/test-dgram-setBroadcast.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-setBroadcast.js)
 - [parallel/test-dgram-setTTL.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-setTTL.js)
-- [parallel/test-dgram-socket-buffer-size.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-socket-buffer-size.js)
 - [parallel/test-dgram-udp6-link-local-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-udp6-link-local-address.js)
 - [parallel/test-dgram-unref-in-cluster.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dgram-unref-in-cluster.js)
 - [parallel/test-diagnostics-channel-http-server-start.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-diagnostics-channel-http-server-start.js)
@@ -544,11 +526,9 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-dns-perf_hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-perf_hooks.js)
 - [parallel/test-dns-resolve-promises.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolve-promises.js)
 - [parallel/test-dns-resolveany-bad-ancount.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolveany-bad-ancount.js)
-- [parallel/test-dns-resolveany.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-resolveany.js)
 - [parallel/test-dns-set-default-order.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-set-default-order.js)
 - [parallel/test-dns-setlocaladdress.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-setlocaladdress.js)
 - [parallel/test-dns-setserver-when-querying.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns-setserver-when-querying.js)
-- [parallel/test-dns.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-dns.js)
 - [parallel/test-domain-abort-on-uncaught.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-abort-on-uncaught.js)
 - [parallel/test-domain-add-remove.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-add-remove.js)
 - [parallel/test-domain-async-id-map-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-domain-async-id-map-leak.js)
@@ -1462,8 +1442,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-net-child-process-connect-reset.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-child-process-connect-reset.js)
 - [parallel/test-net-client-bind-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-client-bind-twice.js)
 - [parallel/test-net-connect-abort-controller.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-abort-controller.js)
-- [parallel/test-net-connect-buffer.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-buffer.js)
-- [parallel/test-net-connect-buffer2.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-buffer2.js)
 - [parallel/test-net-connect-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-keepalive.js)
 - [parallel/test-net-connect-memleak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-memleak.js)
 - [parallel/test-net-connect-nodelay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-connect-nodelay.js)
@@ -1484,7 +1462,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-net-onread-static-buffer.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-onread-static-buffer.js)
 - [parallel/test-net-perf_hooks.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-perf_hooks.js)
 - [parallel/test-net-pingpong.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-pingpong.js)
-- [parallel/test-net-server-call-listen-multiple-times.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-call-listen-multiple-times.js)
 - [parallel/test-net-server-drop-connections.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-drop-connections.js)
 - [parallel/test-net-server-keepalive.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-keepalive.js)
 - [parallel/test-net-server-listen-handle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-listen-handle.js)
@@ -1493,14 +1470,12 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-net-server-nodelay.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-nodelay.js)
 - [parallel/test-net-server-reset.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-reset.js)
 - [parallel/test-net-server-simultaneous-accepts-produce-warning-once.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-simultaneous-accepts-produce-warning-once.js)
-- [parallel/test-net-server-try-ports.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-server-try-ports.js)
 - [parallel/test-net-socket-byteswritten.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-byteswritten.js)
 - [parallel/test-net-socket-connect-invalid-autoselectfamilyattempttimeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-connect-invalid-autoselectfamilyattempttimeout.js)
 - [parallel/test-net-socket-constructor.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-constructor.js)
 - [parallel/test-net-socket-local-address.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-local-address.js)
 - [parallel/test-net-socket-reset-send.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-reset-send.js)
 - [parallel/test-net-socket-reset-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-reset-twice.js)
-- [parallel/test-net-socket-timeout.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-socket-timeout.js)
 - [parallel/test-net-stream.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-stream.js)
 - [parallel/test-net-throttle.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-throttle.js)
 - [parallel/test-net-write-after-close.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-net-write-after-close.js)
@@ -1662,7 +1637,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-readline-input-onerror.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-input-onerror.js)
 - [parallel/test-readline-interface-no-trailing-newline.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface-no-trailing-newline.js)
 - [parallel/test-readline-interface-recursive-writes.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface-recursive-writes.js)
-- [parallel/test-readline-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-interface.js)
 - [parallel/test-readline-promises-interface.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-promises-interface.js)
 - [parallel/test-readline-promises-tab-complete.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-promises-tab-complete.js)
 - [parallel/test-readline-tab-complete.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-readline-tab-complete.js)
@@ -2103,7 +2077,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-tracing-no-crash.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tracing-no-crash.js)
 - [parallel/test-tty-backwards-api.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tty-backwards-api.js)
 - [parallel/test-tty-stdin-pipe.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-tty-stdin-pipe.js)
-- [parallel/test-ttywrap-invalid-fd.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-ttywrap-invalid-fd.js)
 - [parallel/test-ttywrap-stack.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-ttywrap-stack.js)
 - [parallel/test-unhandled-exception-rethrow-error.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-unhandled-exception-rethrow-error.js)
 - [parallel/test-unhandled-exception-with-worker-inuse.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-unhandled-exception-with-worker-inuse.js)
@@ -2126,7 +2099,6 @@ NOTE: This file should not be manually edited. Please edit `tests/node_compat/co
 - [parallel/test-v8-flag-type-check.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-flag-type-check.js)
 - [parallel/test-v8-flags.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-flags.js)
 - [parallel/test-v8-getheapsnapshot-twice.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-getheapsnapshot-twice.js)
-- [parallel/test-v8-serdes.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-serdes.js)
 - [parallel/test-v8-serialize-leak.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-serialize-leak.js)
 - [parallel/test-v8-startup-snapshot-api.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-startup-snapshot-api.js)
 - [parallel/test-v8-stats.js](https://github.com/nodejs/node/tree/v20.11.1/test/parallel/test-v8-stats.js)

tests/node_compat/runner/challenge_new_test.ts

@@ -0,0 +1,72 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+// deno-lint-ignore-file no-console
+
+import { deadline } from "@std/async/deadline";
+import { ensureDir } from "@std/fs/ensure-dir";
+import { copy } from "@std/fs/copy";
+import { withoutAll } from "@std/collections/without-all";
+import {
+  getDenoTests,
+  getNodeTests,
+  NODE_COMPAT_TEST_DEST_URL,
+  runNodeCompatTestCase,
+  VENDORED_NODE_TEST,
+} from "../common.ts";
+import { fromFileUrl } from "@std/path/from-file-url";
+
+/** The timeout ms for single test execution. If a single test didn't finish in this timeout milliseconds, the test is considered as failure */
+const TIMEOUT = 2000;
+
+async function main() {
+  const remainingTests = withoutAll(await getNodeTests(), await getDenoTests());
+
+  console.log(`Remaining tests: ${remainingTests.length}`);
+  const success = [] as string[];
+  let i = 0;
+
+  Deno.addSignalListener("SIGINT", () => {
+    console.log(`Success: ${success.length}`);
+    for (const testPath of success) {
+      console.log(testPath);
+    }
+    Deno.exit(1);
+  });
+
+  for (const testPath of remainingTests) {
+    i++;
+    const source = new URL(testPath, VENDORED_NODE_TEST);
+    const dest = new URL(testPath, NODE_COMPAT_TEST_DEST_URL);
+
+    await ensureDir(new URL(".", dest));
+    await copy(source, dest);
+    const num = String(i).padStart(4, " ");
+    try {
+      const cp = await runNodeCompatTestCase(
+        fromFileUrl(dest),
+        AbortSignal.timeout(TIMEOUT),
+      );
+      const result = await deadline(cp.output(), TIMEOUT + 1000);
+      if (result.code === 0) {
+        console.log(`${num} %cPASS`, "color: green", testPath);
+        success.push(testPath);
+      } else {
+        console.log(`${num} %cFAIL`, "color: red", testPath);
+      }
+    } catch (e) {
+      if (e instanceof DOMException && e.name === "TimeoutError") {
+        console.log(`${num} %cFAIL`, "color: red", testPath);
+      } else {
+        console.log(`Unexpected Error`, e);
+      }
+    } finally {
+      await Deno.remove(dest);
+    }
+  }
+  console.log(`Success: ${success.length}`);
+  for (const testPath of success) {
+    console.log(testPath);
+  }
+  Deno.exit(0);
+}
+
+await main();

tests/node_compat/runner/setup.ts

@@ -10,63 +10,20 @@ import { SEPARATOR } from "@std/path/constants";
 import { ensureFile } from "@std/fs/ensure-file";
 import { writeAll } from "@std/io/write-all";
 import { withoutAll } from "@std/collections/without-all";
-import { relative } from "@std/path/posix/relative";
 import { version } from "./suite/node_version.ts";
-
+import {
-import { config, ignoreList } from "../common.ts";
+  config,
+  getDenoTests,
+  getNodeTests,
+  ignoreList,
+  NODE_COMPAT_TEST_DEST_URL,
+  VENDORED_NODE_TEST,
+} from "../common.ts";
 
 const encoder = new TextEncoder();
 
 const NODE_VERSION = version;
 
-export const NODE_IGNORED_TEST_DIRS = [
-  "addons",
-  "async-hooks",
-  "cctest",
-  "common",
-  "doctool",
-  "embedding",
-  "fixtures",
-  "fuzzers",
-  "js-native-api",
-  "node-api",
-  "overlapped-checker",
-  "report",
-  "testpy",
-  "tick-processor",
-  "tools",
-  "v8-updates",
-  "wasi",
-  "wpt",
-];
-
-export const VENDORED_NODE_TEST = new URL("./suite/test/", import.meta.url);
-export const NODE_COMPAT_TEST_DEST_URL = new URL(
-  "../test/",
-  import.meta.url,
-);
-
-export async function getNodeTests(): Promise<string[]> {
-  const paths: string[] = [];
-  const rootPath = VENDORED_NODE_TEST.href.slice(7);
-  for await (
-    const item of walk(VENDORED_NODE_TEST, { exts: [".js"] })
-  ) {
-    const path = relative(rootPath, item.path);
-    if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) {
-      paths.push(path);
-    }
-  }
-
-  return paths.sort();
-}
-
-export function getDenoTests() {
-  return Object.entries(config.tests)
-    .filter(([testDir]) => !NODE_IGNORED_TEST_DIRS.includes(testDir))
-    .flatMap(([testDir, tests]) => tests.map((test) => testDir + "/" + test));
-}
-
 async function updateToDo() {
   using file = await Deno.open(new URL("./TODO.md", import.meta.url), {
     write: true,
@@ -75,7 +32,7 @@ async function updateToDo() {
   });
 
   const nodeTests = await getNodeTests();
-  const portedTests = getDenoTests();
+  const portedTests = await getDenoTests();
   const remainingTests = withoutAll(nodeTests, portedTests);
   const numPorted = portedTests.length;
   const numMissing = remainingTests.length;
@@ -167,7 +124,9 @@ await copyTests();
 await updateToDo();
 
 if (Deno.args[0] === "--check") {
-  const cmd = new Deno.Command("git", { args: ["status", "-s"] });
+  const cmd = new Deno.Command("git", {
+    args: ["status", "-s", "tests/node_compat/test"],
+  });
   const { stdout } = await cmd.output();
 
   if (stdout.length > 0) {

tests/node_compat/test.ts

@@ -24,6 +24,7 @@ import {
   config,
   getPathsFromTestSuites,
   partitionParallelTestPaths,
+  runNodeCompatTestCase,
 } from "./common.ts";
 
 // If the test case is invoked like
@@ -40,7 +41,6 @@ const testPaths = partitionParallelTestPaths(
 testPaths.sequential = distinct(testPaths.sequential);
 testPaths.parallel = distinct(testPaths.parallel);
 
-const cwd = new URL(".", import.meta.url);
 const windowsIgnorePaths = new Set(
   getPathsFromTestSuites(config.windowsIgnore),
 );
@@ -49,13 +49,6 @@ const darwinIgnorePaths = new Set(
 );
 
 const decoder = new TextDecoder();
-let testSerialId = 0;
-
-function parseFlags(source: string): string[] {
-  const line = /^\/\/ Flags: (.+)$/um.exec(source);
-  if (line == null) return [];
-  return line[1].split(" ");
-}
 
 async function runTest(t: Deno.TestContext, path: string): Promise<void> {
   // If filter patterns are given and any pattern doesn't match
@@ -77,60 +70,7 @@ async function runTest(t: Deno.TestContext, path: string): Promise<void> {
     sanitizeExit: false,
     fn: async () => {
       const testCase = join(toolsPath, "test", path);
-
+      const command = await runNodeCompatTestCase(testCase);
-      const v8Flags = ["--stack-size=4000"];
-      const testSource = await Deno.readTextFile(testCase);
-      const envVars: Record<string, string> = {};
-      const knownGlobals: string[] = [];
-      parseFlags(testSource).forEach((flag) => {
-        switch (flag) {
-          case "--expose_externalize_string":
-            v8Flags.push("--expose-externalize-string");
-            knownGlobals.push("createExternalizableString");
-            break;
-          case "--expose-gc":
-            v8Flags.push("--expose-gc");
-            knownGlobals.push("gc");
-            break;
-          default:
-            break;
-        }
-      });
-      if (knownGlobals.length > 0) {
-        envVars["NODE_TEST_KNOWN_GLOBALS"] = knownGlobals.join(",");
-      }
-      // TODO(nathanwhit): once we match node's behavior on executing
-      // `node:test` tests when we run a file, we can remove this
-      const usesNodeTest = testSource.includes("node:test");
-      const args = [
-        usesNodeTest ? "test" : "run",
-        "-A",
-        "--quiet",
-        //"--unsafely-ignore-certificate-errors",
-        "--unstable-unsafe-proto",
-        "--unstable-bare-node-builtins",
-        "--unstable-fs",
-        "--v8-flags=" + v8Flags.join(),
-      ];
-      if (usesNodeTest) {
-        // deno test typechecks by default + we want to pass script args
-        args.push("--no-check", "runner.ts", "--", testCase);
-      } else {
-        args.push("runner.ts", testCase);
-      }
-
-      // Pipe stdout in order to output each test result as Deno.test output
-      // That way the tests will respect the `--quiet` option when provided
-      const command = new Deno.Command(Deno.execPath(), {
-        args,
-        env: {
-          TEST_SERIAL_ID: String(testSerialId++),
-          ...envVars,
-        },
-        cwd,
-        stdout: "piped",
-        stderr: "piped",
-      }).spawn();
       const warner = setTimeout(() => {
         console.error(`Test is running slow: ${testCase}`);
       }, 2 * 60_000);