From e49f0bee57b275e785107799796e0002a3e99fbc Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 11 Jan 2023 08:23:08 -0500 Subject: [PATCH] chore(ci): checkout less submodules based on job (#17343) Co-authored-by: Luca Casonato --- .github/mtime_cache/action.js | 5 +- .github/workflows/ci.generate.ts | 267 ++++++++++++++++++------------- .github/workflows/ci.yml | 26 +-- 3 files changed, 178 insertions(+), 120 deletions(-) diff --git a/.github/mtime_cache/action.js b/.github/mtime_cache/action.js index 803e9b7102..37b957a742 100644 --- a/.github/mtime_cache/action.js +++ b/.github/mtime_cache/action.js @@ -151,7 +151,10 @@ async function* ls(dir = "") { case "120000": // Symbolic link. break; case "160000": // Git submodule. - yield* ls(path); + // sometimes we don't checkout all submodules + if (fs.existsSync(path)) { + yield* ls(path); + } break; default: { // Regular file. diff --git a/.github/workflows/ci.generate.ts b/.github/workflows/ci.generate.ts index 38120bc650..d5807605df 100644 --- a/.github/workflows/ci.generate.ts +++ b/.github/workflows/ci.generate.ts @@ -2,6 +2,109 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. import * as yaml from "https://deno.land/std@0.171.0/encoding/yaml.ts"; +const Runners = { + linux: + "${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}", + macos: "macos-12", + windows: + "${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}", +}; + +const sysRootStep = { + name: "Set up incremental LTO and sysroot build", + run: `# Avoid running man-db triggers, which sometimes takes several minutes +# to complete. +sudo apt-get remove --purge -y man-db + +# Install clang-15, lld-15, and debootstrap. +echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" | + sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-focal-15.list +curl https://apt.llvm.org/llvm-snapshot.gpg.key | + gpg --dearmor | +sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg +sudo apt-get update +sudo apt-get install --no-install-recommends debootstrap \\ + clang-15 lld-15 + +# Create ubuntu-16.04 sysroot environment, which is used to avoid +# depending on a very recent version of glibc. +# \`libc6-dev\` is required for building any C source files. +# \`file\` and \`make\` are needed to build libffi-sys. +# \`curl\` is needed to build rusty_v8. +sudo debootstrap \\ + --include=ca-certificates,curl,file,libc6-dev,make \\ + --no-merged-usr --variant=minbase xenial /sysroot \\ + http://azure.archive.ubuntu.com/ubuntu +sudo mount --rbind /dev /sysroot/dev +sudo mount --rbind /sys /sysroot/sys +sudo mount --rbind /home /sysroot/home +sudo mount -t proc /proc /sysroot/proc + +# Configure the build environment. Both Rust and Clang will produce +# llvm bitcode only, so we can use lld's incremental LTO support. +cat >> $GITHUB_ENV << __0 +CARGO_PROFILE_BENCH_INCREMENTAL=false +CARGO_PROFILE_BENCH_LTO=false +CARGO_PROFILE_RELEASE_INCREMENTAL=false +CARGO_PROFILE_RELEASE_LTO=false +RUSTFLAGS<<__1 + -C linker-plugin-lto=true + -C linker=clang-15 + -C link-arg=-fuse-ld=lld-15 + -C link-arg=--sysroot=/sysroot + -C link-arg=-Wl,--allow-shlib-undefined + -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache + -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m + \${{ env.RUSTFLAGS }} +__1 +RUSTDOCFLAGS<<__1 + -C linker-plugin-lto=true + -C linker=clang-15 + -C link-arg=-fuse-ld=lld-15 + -C link-arg=--sysroot=/sysroot + -C link-arg=-Wl,--allow-shlib-undefined + -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache + -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m + \${{ env.RUSTFLAGS }} +__1 +CC=clang-15 +CFLAGS=-flto=thin --sysroot=/sysroot +__0`, +}; + +const submoduleStep = (submodule: string) => ({ + name: `Clone submodule ${submodule}`, + run: `git submodule update --init --recursive --depth=1 -- ${submodule}`, +}); + +const installRustStep = { + uses: "dtolnay/rust-toolchain@stable", +}; +const installPythonSteps = [{ + name: "Install Python", + uses: "actions/setup-python@v4", + with: { "python-version": 3.8 }, +}, { + name: "Remove unused versions of Python", + if: "startsWith(matrix.os, 'windows')", + run: [ + '$env:PATH -split ";" |', + ' Where-Object { Test-Path "$_\\python.exe" } |', + " Select-Object -Skip 1 |", + ' ForEach-Object { Move-Item "$_" "$_.disabled" }', + ].join("\n"), +}]; +const installNodeStep = { + name: "Install Node", + uses: "actions/setup-node@v3", + with: { "node-version": 17 }, +}; +const installDenoStep = { + name: "Install Deno", + uses: "denoland/setup-deno@v1", + with: { "deno-version": "v1.x" }, +}; + const ci = { name: "ci", on: ["push", "pull_request"], @@ -23,51 +126,45 @@ const ci = { matrix: { include: [ { - os: "macos-12", + os: Runners.macos, job: "test", profile: "fastci", }, { - os: "macos-12", + os: Runners.macos, job: "test", profile: "release", }, { - os: - "${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}", + os: Runners.windows, job: "test", profile: "fastci", }, { - os: - "${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}", + os: Runners.windows, job: "test", profile: "release", }, { - os: - "${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}", + os: Runners.linux, job: "test", profile: "release", use_sysroot: true, }, { - os: - "${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}", + os: Runners.linux, job: "bench", profile: "release", use_sysroot: true, }, { - os: - "${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}", + os: Runners.linux, job: "test", profile: "debug", use_sysroot: true, }, { - os: - "${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}", + os: Runners.linux, job: "lint", profile: "debug", }, @@ -101,9 +198,18 @@ const ci = { // other commits have landed it will become impossible to rebuild if // the checkout is too shallow. "fetch-depth": 5, - submodules: "recursive", + submodules: false, }, }, + submoduleStep("./test_util/std"), + { + ...submoduleStep("./test_util/wpt"), + if: "matrix.job == 'test'", + }, + { + ...submoduleStep("./third_party"), + if: "matrix.job == 'lint' || matrix.job == 'bench'", + }, { name: "Create source tarballs (release, linux)", if: [ @@ -119,33 +225,13 @@ const ci = { " -czvf target/release/deno_src.tar.gz -C .. deno", ].join("\n"), }, - { uses: "dtolnay/rust-toolchain@stable" }, + installRustStep, { - name: "Install Deno", if: "matrix.job == 'lint' || matrix.job == 'test'", - uses: "denoland/setup-deno@v1", - with: { "deno-version": "v1.x" }, - }, - { - name: "Install Python", - uses: "actions/setup-python@v4", - with: { "python-version": 3.8 }, - }, - { - name: "Install Node", - uses: "actions/setup-node@v3", - with: { "node-version": 17 }, - }, - { - name: "Remove unused versions of Python", - if: "startsWith(matrix.os, 'windows')", - run: [ - '$env:PATH -split ";" |', - ' Where-Object { Test-Path "$_\\python.exe" } |', - " Select-Object -Skip 1 |", - ' ForEach-Object { Move-Item "$_" "$_.disabled" }', - ].join("\n"), + ...installDenoStep, }, + ...installPythonSteps, + installNodeStep, { name: "Setup gcloud (unix)", if: [ @@ -174,7 +260,9 @@ const ci = { "startsWith(github.ref, 'refs/tags/'))", ].join("\n"), uses: "google-github-actions/setup-gcloud@v0", - env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" }, + env: { + CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe", + }, with: { project_id: "denoland", service_account_key: "${{ secrets.GCP_SA_KEY }}", @@ -193,68 +281,8 @@ const ci = { run: 'echo "DENO_CANARY=true" >> $GITHUB_ENV', }, { - name: "Set up incremental LTO and sysroot build", if: "matrix.use_sysroot", - run: [ - "# Avoid running man-db triggers, which sometimes takes several minutes", - "# to complete.", - "sudo apt-get remove --purge -y man-db", - "", - "# Install clang-15, lld-15, and debootstrap.", - 'echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" |', - " sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-focal-15.list", - "curl https://apt.llvm.org/llvm-snapshot.gpg.key |", - " gpg --dearmor |", - "sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg", - "sudo apt-get update", - "sudo apt-get install --no-install-recommends debootstrap \\", - " clang-15 lld-15", - "", - "# Create ubuntu-16.04 sysroot environment, which is used to avoid", - "# depending on a very recent version of glibc.", - "# `libc6-dev` is required for building any C source files.", - "# `file` and `make` are needed to build libffi-sys.", - "# `curl` is needed to build rusty_v8.", - "sudo debootstrap \\", - " --include=ca-certificates,curl,file,libc6-dev,make \\", - " --no-merged-usr --variant=minbase xenial /sysroot \\", - " http://azure.archive.ubuntu.com/ubuntu", - "sudo mount --rbind /dev /sysroot/dev", - "sudo mount --rbind /sys /sysroot/sys", - "sudo mount --rbind /home /sysroot/home", - "sudo mount -t proc /proc /sysroot/proc", - "", - "# Configure the build environment. Both Rust and Clang will produce", - "# llvm bitcode only, so we can use lld's incremental LTO support.", - "cat >> $GITHUB_ENV << __0", - "CARGO_PROFILE_BENCH_INCREMENTAL=false", - "CARGO_PROFILE_BENCH_LTO=false", - "CARGO_PROFILE_RELEASE_INCREMENTAL=false", - "CARGO_PROFILE_RELEASE_LTO=false", - "RUSTFLAGS<<__1", - " -C linker-plugin-lto=true", - " -C linker=clang-15", - " -C link-arg=-fuse-ld=lld-15", - " -C link-arg=--sysroot=/sysroot", - " -C link-arg=-Wl,--allow-shlib-undefined", - " -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache", - " -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m", - " ${{ env.RUSTFLAGS }}", - "__1", - "RUSTDOCFLAGS<<__1", - " -C linker-plugin-lto=true", - " -C linker=clang-15", - " -C link-arg=-fuse-ld=lld-15", - " -C link-arg=--sysroot=/sysroot", - " -C link-arg=-Wl,--allow-shlib-undefined", - " -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache", - " -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m", - " ${{ env.RUSTFLAGS }}", - "__1", - "CC=clang-15", - "CFLAGS=-flto=thin --sysroot=/sysroot", - "__0", - ].join("\n"), + ...sysRootStep, }, { name: "Log versions", @@ -455,7 +483,9 @@ const ci = { "github.repository == 'denoland/deno' &&", "github.ref == 'refs/heads/main'", ].join("\n"), - env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" }, + env: { + CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe", + }, shell: "bash", run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/', @@ -472,7 +502,9 @@ const ci = { name: "Test fastci", if: "(matrix.job == 'test' && matrix.profile == 'fastci')", run: "cargo test --locked", - env: { CARGO_PROFILE_DEV_DEBUG: 0 }, + env: { + CARGO_PROFILE_DEV_DEBUG: 0, + }, }, { name: "Test release", @@ -492,7 +524,9 @@ const ci = { "matrix.profile == 'release' && startsWith(github.ref, 'refs/tags/')", shell: "bash", run: 'target/release/deno eval "console.log(1+2)" | grep 3', - env: { NO_COLOR: 1 }, + env: { + NO_COLOR: 1, + }, }, { // Verify that the binary actually works in the Ubuntu-16.04 sysroot. @@ -515,7 +549,9 @@ const ci = { "matrix.profile == 'debug' &&", "github.ref == 'refs/heads/main'", ].join("\n"), - env: { DENO_BIN: "./target/debug/deno" }, + env: { + DENO_BIN: "./target/debug/deno", + }, run: [ "deno run --allow-env --allow-net --allow-read --allow-run \\", " --allow-write --unstable \\", @@ -533,7 +569,9 @@ const ci = { "startsWith(matrix.os, 'ubuntu') && matrix.job == 'test' &&", "matrix.profile == 'release' && !startsWith(github.ref, 'refs/tags/')", ].join("\n"), - env: { DENO_BIN: "./target/release/deno" }, + env: { + DENO_BIN: "./target/release/deno", + }, run: [ "deno run --allow-env --allow-net --allow-read --allow-run \\", " --allow-write --unstable \\", @@ -598,7 +636,9 @@ const ci = { "github.repository == 'denoland/deno' &&", "github.ref == 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')", ].join("\n"), - env: { DENOBOT_PAT: "${{ secrets.DENOBOT_PAT }}" }, + env: { + DENOBOT_PAT: "${{ secrets.DENOBOT_PAT }}", + }, run: [ "git clone --depth 1 --branch gh-pages \\", " https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git \\", @@ -625,7 +665,10 @@ const ci = { { name: "Worker info", if: "matrix.job == 'bench'", - run: ["cat /proc/cpuinfo", "cat /proc/meminfo"].join("\n"), + run: [ + "cat /proc/cpuinfo", + "cat /proc/meminfo", + ].join("\n"), }, { name: "Upload release to dl.deno.land (unix)", @@ -648,7 +691,9 @@ const ci = { "github.repository == 'denoland/deno' &&", "startsWith(github.ref, 'refs/tags/')", ].join("\n"), - env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" }, + env: { + CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe", + }, shell: "bash", run: 'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/', @@ -676,7 +721,9 @@ const ci = { "github.repository == 'denoland/deno' &&", "startsWith(github.ref, 'refs/tags/')", ].join("\n"), - env: { GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" }, + env: { + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}", + }, with: { files: [ "target/release/deno-x86_64-pc-windows-msvc.zip", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9861e53ea8..4548a7fc21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,15 @@ jobs: uses: actions/checkout@v3 with: fetch-depth: 5 - submodules: recursive + submodules: false + - name: Clone submodule ./test_util/std + run: git submodule update --init --recursive --depth=1 -- ./test_util/std + - name: Clone submodule ./test_util/wpt + run: git submodule update --init --recursive --depth=1 -- ./test_util/wpt + if: matrix.job == 'test' + - name: Clone submodule ./third_party + run: git submodule update --init --recursive --depth=1 -- ./third_party + if: matrix.job == 'lint' || matrix.job == 'bench' - name: 'Create source tarballs (release, linux)' if: |- startsWith(matrix.os, 'ubuntu') && @@ -71,8 +79,8 @@ jobs: tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \ -czvf target/release/deno_src.tar.gz -C .. deno - uses: dtolnay/rust-toolchain@stable - - name: Install Deno - if: matrix.job == 'lint' || matrix.job == 'test' + - if: matrix.job == 'lint' || matrix.job == 'test' + name: Install Deno uses: denoland/setup-deno@v1 with: deno-version: v1.x @@ -80,10 +88,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: 3.8 - - name: Install Node - uses: actions/setup-node@v3 - with: - node-version: 17 - name: Remove unused versions of Python if: 'startsWith(matrix.os, ''windows'')' run: |- @@ -91,6 +95,10 @@ jobs: Where-Object { Test-Path "$_\python.exe" } | Select-Object -Skip 1 | ForEach-Object { Move-Item "$_" "$_.disabled" } + - name: Install Node + uses: actions/setup-node@v3 + with: + node-version: 17 - name: Setup gcloud (unix) if: |- runner.os != 'Windows' && @@ -127,8 +135,8 @@ jobs: github.ref == 'refs/heads/main' shell: bash run: echo "DENO_CANARY=true" >> $GITHUB_ENV - - name: Set up incremental LTO and sysroot build - if: matrix.use_sysroot + - if: matrix.use_sysroot + name: Set up incremental LTO and sysroot build run: |- # Avoid running man-db triggers, which sometimes takes several minutes # to complete.