From 6e9c95785d252d88979ea6226e05198611d9125b Mon Sep 17 00:00:00 2001 From: Yoshiya Hinosawa Date: Mon, 15 May 2023 21:22:53 +0900 Subject: [PATCH] chore: check node_compat config diff in CI (#19119) --- .dprint.json | 2 +- .github/workflows/ci.generate.ts | 10 +++ .github/workflows/ci.yml | 6 ++ .gitmodules | 3 + tools/node_compat/TODO.md | 3 +- tools/node_compat/node | 1 + tools/node_compat/setup.ts | 117 ++++++------------------------- 7 files changed, 45 insertions(+), 97 deletions(-) create mode 160000 tools/node_compat/node diff --git a/.dprint.json b/.dprint.json index e1caea7fe3..31f945acda 100644 --- a/.dprint.json +++ b/.dprint.json @@ -48,7 +48,7 @@ "test_util/wpt", "third_party", "tools/node_compat/TODO.md", - "tools/node_compat/versions", + "tools/node_compat/node", "tools/wpt/expectation.json", "tools/wpt/manifest.json", "ext/websocket/autobahn/reports" diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 50c7b54b7b..84ff65b4c6 100755 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -362,6 +362,10 @@ const ci = { ...submoduleStep("./test_util/wpt"), if: "matrix.wpt", }, + { + ...submoduleStep("./tools/node_compat/node"), + if: "matrix.job == 'lint'", + }, { name: "Create source tarballs (release, linux)", if: [ @@ -541,6 +545,12 @@ const ci = { run: "deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js", }, + { + name: "node_compat/setup.ts --check", + if: "matrix.job == 'lint'", + run: + "deno run --allow-write --allow-read --allow-run=git ./tools/node_compat/setup.ts --check", + }, { name: "Build debug", if: "matrix.job == 'test' && matrix.profile == 'debug'", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c266f8f40a..9679ffbb63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,6 +120,9 @@ jobs: - name: Clone submodule ./test_util/wpt run: git submodule update --init --recursive --depth=1 -- ./test_util/wpt if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.wpt)' + - name: Clone submodule ./tools/node_compat/node + run: git submodule update --init --recursive --depth=1 -- ./tools/node_compat/node + if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')' - name: 'Create source tarballs (release, linux)' if: |- !(github.event_name == 'pull_request' && matrix.skip_pr) && (startsWith(matrix.os, 'ubuntu') && @@ -328,6 +331,9 @@ jobs: - name: lint.js if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')' run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js + - name: node_compat/setup.ts --check + if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''lint'')' + run: deno run --allow-write --allow-read --allow-run=git ./tools/node_compat/setup.ts --check - name: Build debug if: '!(github.event_name == ''pull_request'' && matrix.skip_pr) && (matrix.job == ''test'' && matrix.profile == ''debug'')' run: cargo build --locked --all-targets diff --git a/.gitmodules b/.gitmodules index 9e4f12afa6..a81f84fc6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ path = test_util/wpt url = https://github.com/web-platform-tests/wpt.git +[submodule "tools/node_compat/node"] + path = tools/node_compat/node + url = https://github.com/denoland/node_test.git diff --git a/tools/node_compat/TODO.md b/tools/node_compat/TODO.md index 650cd4b165..2ea6983f91 100644 --- a/tools/node_compat/TODO.md +++ b/tools/node_compat/TODO.md @@ -3,7 +3,7 @@ NOTE: This file should not be manually edited. Please edit 'cli/tests/node_compat/config.json' and run 'tools/node_compat/setup.ts' instead. -Total: 2934 +Total: 2935 - [abort/test-abort-backtrace.js](https://github.com/nodejs/node/tree/v18.12.1/test/abort/test-abort-backtrace.js) - [abort/test-abort-fatal-error.js](https://github.com/nodejs/node/tree/v18.12.1/test/abort/test-abort-fatal-error.js) @@ -1815,6 +1815,7 @@ Total: 2934 - [parallel/test-process-env.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-env.js) - [parallel/test-process-euid-egid.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-euid-egid.js) - [parallel/test-process-exception-capture-errors.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-errors.js) +- [parallel/test-process-exception-capture-should-abort-on-uncaught-setflagsfromstring.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-should-abort-on-uncaught-setflagsfromstring.js) - [parallel/test-process-exception-capture-should-abort-on-uncaught.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture-should-abort-on-uncaught.js) - [parallel/test-process-exception-capture.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exception-capture.js) - [parallel/test-process-exec-argv.js](https://github.com/nodejs/node/tree/v18.12.1/test/parallel/test-process-exec-argv.js) diff --git a/tools/node_compat/node b/tools/node_compat/node new file mode 160000 index 0000000000..d0d9c1ba9d --- /dev/null +++ b/tools/node_compat/node @@ -0,0 +1 @@ +Subproject commit d0d9c1ba9d3facf1086438e21d6d329c599e5a3b diff --git a/tools/node_compat/setup.ts b/tools/node_compat/setup.ts index c8fd6a8e09..5bd5a5ba56 100755 --- a/tools/node_compat/setup.ts +++ b/tools/node_compat/setup.ts @@ -1,23 +1,11 @@ -#!/usr/bin/env -S deno run --allow-read=. --allow-write=. --allow-net=nodejs.org +#!/usr/bin/env -S deno run --allow-read=. --allow-write=. --allow-run=git // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -/** This script downloads Node.js source tarball, extracts it and copies the - * test files according to the config file `cli/tests/node_compat/config.json` - */ +/** This copies the test files according to the config file `cli/tests/node_compat/config.jsonc` */ -import { Foras, gunzip } from "https://deno.land/x/denoflate@2.0.2/deno/mod.ts"; -import { Untar } from "../../test_util/std/archive/untar.ts"; import { walk } from "../../test_util/std/fs/walk.ts"; -import { - dirname, - fromFileUrl, - join, - sep, -} from "../../test_util/std/path/mod.ts"; +import { sep } from "../../test_util/std/path/mod.ts"; import { ensureFile } from "../../test_util/std/fs/ensure_file.ts"; -import { Buffer } from "../../test_util/std/io/buffer.ts"; -import { copy } from "../../test_util/std/streams/copy.ts"; -import { readAll } from "../../test_util/std/streams/read_all.ts"; import { writeAll } from "../../test_util/std/streams/write_all.ts"; import { withoutAll } from "../../test_util/std/collections/without_all.ts"; import { relative } from "../../test_util/std/path/posix.ts"; @@ -27,8 +15,6 @@ import { config, ignoreList } from "../../cli/tests/node_compat/common.ts"; const encoder = new TextEncoder(); const NODE_VERSION = config.nodeVersion; -const NODE_NAME = "node-v" + NODE_VERSION; -const NODE_ARCHIVE_NAME = `${NODE_NAME}.tar.gz`; const NODE_IGNORED_TEST_DIRS = [ "addons", @@ -51,25 +37,17 @@ const NODE_IGNORED_TEST_DIRS = [ "wpt", ]; -const NODE_TARBALL_URL = - `https://nodejs.org/dist/v${NODE_VERSION}/${NODE_ARCHIVE_NAME}`; -const NODE_VERSIONS_ROOT = new URL("versions/", import.meta.url); -const NODE_TARBALL_LOCAL_URL = new URL(NODE_ARCHIVE_NAME, NODE_VERSIONS_ROOT); -// local dir url where we copy the node tests -const NODE_LOCAL_ROOT_URL = new URL(NODE_NAME, NODE_VERSIONS_ROOT); -const NODE_LOCAL_TEST_URL = new URL(NODE_NAME + "/test/", NODE_VERSIONS_ROOT); +const VENDORED_NODE_TEST = new URL("node/test/", import.meta.url); const NODE_COMPAT_TEST_DEST_URL = new URL( "../../cli/tests/node_compat/test/", import.meta.url, ); -Foras.initSyncBundledOnce(); - async function getNodeTests(): Promise { const paths: string[] = []; - const rootPath = NODE_LOCAL_TEST_URL.href.slice(7); + const rootPath = VENDORED_NODE_TEST.href.slice(7); for await ( - const item of walk(NODE_LOCAL_TEST_URL, { exts: [".js"] }) + const item of walk(VENDORED_NODE_TEST, { exts: [".js"] }) ) { const path = relative(rootPath, item.path); if (NODE_IGNORED_TEST_DIRS.every((dir) => !path.startsWith(dir))) { @@ -125,33 +103,6 @@ async function clearTests() { } } -async function decompressTests() { - console.log(`Decompressing ${NODE_ARCHIVE_NAME}...`); - - const compressedFile = await Deno.open(NODE_TARBALL_LOCAL_URL); - - const buffer = new Buffer(gunzip(await readAll(compressedFile))); - compressedFile.close(); - - const tar = new Untar(buffer); - const outFolder = dirname(fromFileUrl(NODE_TARBALL_LOCAL_URL)); - const testsFolder = `${NODE_NAME}/test`; - - for await (const entry of tar) { - if (entry.type !== "file") continue; - if (!entry.fileName.startsWith(testsFolder)) continue; - const path = join(outFolder, entry.fileName); - await ensureFile(path); - const file = await Deno.open(path, { - create: true, - truncate: true, - write: true, - }); - await copy(entry, file); - file.close(); - } -} - /** Checks if file has entry in config.json */ function hasEntry(file: string, suite: string) { return Array.isArray(config.tests[suite]) && @@ -161,12 +112,12 @@ function hasEntry(file: string, suite: string) { async function copyTests() { console.log("Copying test files..."); - for await (const entry of walk(NODE_LOCAL_TEST_URL, { skip: ignoreList })) { + for await (const entry of walk(VENDORED_NODE_TEST, { skip: ignoreList })) { const fragments = entry.path.split(sep); // suite is the directory name after test/. For example, if the file is - // "node-v18.12.1/test/fixtures/policy/main.mjs" + // "node_comapt/node/test/fixtures/policy/main.mjs" // then suite is "fixtures/policy" - const suite = fragments.slice(fragments.indexOf(NODE_NAME) + 2, -1) + const suite = fragments.slice(fragments.indexOf("node_compat") + 3, -1) .join("/"); if (!hasEntry(entry.name, suite)) { continue; @@ -180,8 +131,9 @@ async function copyTests() { write: true, }); const srcFile = await Deno.open( - new URL(`${suite}/${entry.name}`, NODE_LOCAL_TEST_URL), + new URL(`${suite}/${entry.name}`, VENDORED_NODE_TEST), ); + // Add header to js files if (dest.pathname.endsWith("js")) { await writeAll( destFile, @@ -199,44 +151,19 @@ async function copyTests() { } } -/** Downloads Node tarball */ -async function downloadFile() { - console.log( - `Downloading ${NODE_TARBALL_URL} in "${NODE_TARBALL_LOCAL_URL}" ...`, - ); - const response = await fetch(NODE_TARBALL_URL); - if (!response.ok) { - throw new Error(`Request failed with status ${response.status}`); - } - await ensureFile(NODE_TARBALL_LOCAL_URL); - const file = await Deno.open(NODE_TARBALL_LOCAL_URL, { - truncate: true, - write: true, - create: true, - }); - await response.body.pipeTo(file.writable); -} - // main -try { - Deno.lstatSync(NODE_TARBALL_LOCAL_URL); -} catch (e) { - if (!(e instanceof Deno.errors.NotFound)) { - throw e; - } - await downloadFile(); -} - -try { - Deno.lstatSync(NODE_LOCAL_ROOT_URL); -} catch (e) { - if (!(e instanceof Deno.errors.NotFound)) { - throw e; - } - await decompressTests(); -} - await clearTests(); await copyTests(); await updateToDo(); + +if (Deno.args[0] === "--check") { + const cmd = new Deno.Command("git", { args: ["status", "-s"] }); + const { stdout } = await cmd.output(); + + if (stdout.length > 0) { + console.log("The following files have been changed:"); + console.log(new TextDecoder().decode(stdout)); + Deno.exit(1); + } +}