foster/denoland-deno
1
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-11-23 15:16:54 -05:00
Code Issues Packages 1 Wiki Activity

chore(ci): checkout less submodules based on job (#17343)

Browse source 
Co-authored-by: Luca Casonato <lucacasonato@users.noreply.github.com>
This commit is contained in:
David Sherret 2023-01-11 08:23:08 -05:00 committed by David Sherret
parent 42325d0fb6
commit e49f0bee57
3 changed files with 178 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/mtime_cache/action.js vendored
View file

@ -151,7 +151,10 @@ async function* ls(dir = "") {
case "120000": // Symbolic link.
break;
case "160000": // Git submodule.
yield* ls(path);
// sometimes we don't checkout all submodules
if (fs.existsSync(path)) {
yield* ls(path);
}
break;
default: {
// Regular file.

267
.github/workflows/ci.generate.ts vendored
View file

@ -2,6 +2,109 @@
// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
import * as yaml from "https://deno.land/std@0.171.0/encoding/yaml.ts";
const Runners = {
linux:
"${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}",
macos: "macos-12",
windows:
"${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}",
};
const sysRootStep = {
name: "Set up incremental LTO and sysroot build",
run: `# Avoid running man-db triggers, which sometimes takes several minutes
# to complete.
sudo apt-get remove --purge -y man-db
# Install clang-15, lld-15, and debootstrap.
echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" |
sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-focal-15.list
curl https://apt.llvm.org/llvm-snapshot.gpg.key |
gpg --dearmor |
sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
sudo apt-get update
sudo apt-get install --no-install-recommends debootstrap \\
clang-15 lld-15
# Create ubuntu-16.04 sysroot environment, which is used to avoid
# depending on a very recent version of glibc.
# \`libc6-dev\` is required for building any C source files.
# \`file\` and \`make\` are needed to build libffi-sys.
# \`curl\` is needed to build rusty_v8.
sudo debootstrap \\
--include=ca-certificates,curl,file,libc6-dev,make \\
--no-merged-usr --variant=minbase xenial /sysroot \\
http://azure.archive.ubuntu.com/ubuntu
sudo mount --rbind /dev /sysroot/dev
sudo mount --rbind /sys /sysroot/sys
sudo mount --rbind /home /sysroot/home
sudo mount -t proc /proc /sysroot/proc
# Configure the build environment. Both Rust and Clang will produce
# llvm bitcode only, so we can use lld's incremental LTO support.
cat >> $GITHUB_ENV << __0
CARGO_PROFILE_BENCH_INCREMENTAL=false
CARGO_PROFILE_BENCH_LTO=false
CARGO_PROFILE_RELEASE_INCREMENTAL=false
CARGO_PROFILE_RELEASE_LTO=false
RUSTFLAGS<<__1
-C linker-plugin-lto=true
-C linker=clang-15
-C link-arg=-fuse-ld=lld-15
-C link-arg=--sysroot=/sysroot
-C link-arg=-Wl,--allow-shlib-undefined
-C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
-C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
\${{ env.RUSTFLAGS }}
__1
RUSTDOCFLAGS<<__1
-C linker-plugin-lto=true
-C linker=clang-15
-C link-arg=-fuse-ld=lld-15
-C link-arg=--sysroot=/sysroot
-C link-arg=-Wl,--allow-shlib-undefined
-C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
-C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
\${{ env.RUSTFLAGS }}
__1
CC=clang-15
CFLAGS=-flto=thin --sysroot=/sysroot
__0`,
};
const submoduleStep = (submodule: string) => ({
name: `Clone submodule ${submodule}`,
run: `git submodule update --init --recursive --depth=1 -- ${submodule}`,
});
const installRustStep = {
uses: "dtolnay/rust-toolchain@stable",
};
const installPythonSteps = [{
name: "Install Python",
uses: "actions/setup-python@v4",
with: { "python-version": 3.8 },
}, {
name: "Remove unused versions of Python",
if: "startsWith(matrix.os, 'windows')",
run: [
'$env:PATH -split ";" |',
' Where-Object { Test-Path "$_\\python.exe" } |',
" Select-Object -Skip 1 |",
' ForEach-Object { Move-Item "$_" "$_.disabled" }',
].join("\n"),
}];
const installNodeStep = {
name: "Install Node",
uses: "actions/setup-node@v3",
with: { "node-version": 17 },
};
const installDenoStep = {
name: "Install Deno",
uses: "denoland/setup-deno@v1",
with: { "deno-version": "v1.x" },
};
const ci = {
name: "ci",
on: ["push", "pull_request"],
@ -23,51 +126,45 @@ const ci = {
matrix: {
include: [
{
os: "macos-12",
os: Runners.macos,
job: "test",
profile: "fastci",
},
{
os: "macos-12",
os: Runners.macos,
job: "test",
profile: "release",
},
{
os:
"${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}",
os: Runners.windows,
job: "test",
profile: "fastci",
},
{
os:
"${{ github.repository == 'denoland/deno' && 'windows-2019-xl' || 'windows-2019' }}",
os: Runners.windows,
job: "test",
profile: "release",
},
{
os:
"${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}",
os: Runners.linux,
job: "test",
profile: "release",
use_sysroot: true,
},
{
os:
"${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}",
os: Runners.linux,
job: "bench",
profile: "release",
use_sysroot: true,
},
{
os:
"${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}",
os: Runners.linux,
job: "test",
profile: "debug",
use_sysroot: true,
},
{
os:
"${{ github.repository == 'denoland/deno' && 'ubuntu-20.04-xl' || 'ubuntu-20.04' }}",
os: Runners.linux,
job: "lint",
profile: "debug",
},
@ -101,9 +198,18 @@ const ci = {
// other commits have landed it will become impossible to rebuild if
// the checkout is too shallow.
"fetch-depth": 5,
submodules: "recursive",
submodules: false,
},
},
submoduleStep("./test_util/std"),
{
...submoduleStep("./test_util/wpt"),
if: "matrix.job == 'test'",
},
{
...submoduleStep("./third_party"),
if: "matrix.job == 'lint' || matrix.job == 'bench'",
},
{
name: "Create source tarballs (release, linux)",
if: [
@ -119,33 +225,13 @@ const ci = {
" -czvf target/release/deno_src.tar.gz -C .. deno",
].join("\n"),
},
{ uses: "dtolnay/rust-toolchain@stable" },
installRustStep,
{
name: "Install Deno",
if: "matrix.job == 'lint' || matrix.job == 'test'",
uses: "denoland/setup-deno@v1",
with: { "deno-version": "v1.x" },
},
{
name: "Install Python",
uses: "actions/setup-python@v4",
with: { "python-version": 3.8 },
},
{
name: "Install Node",
uses: "actions/setup-node@v3",
with: { "node-version": 17 },
},
{
name: "Remove unused versions of Python",
if: "startsWith(matrix.os, 'windows')",
run: [
'$env:PATH -split ";" |',
' Where-Object { Test-Path "$_\\python.exe" } |',
" Select-Object -Skip 1 |",
' ForEach-Object { Move-Item "$_" "$_.disabled" }',
].join("\n"),
...installDenoStep,
},
...installPythonSteps,
installNodeStep,
{
name: "Setup gcloud (unix)",
if: [
@ -174,7 +260,9 @@ const ci = {
"startsWith(github.ref, 'refs/tags/'))",
].join("\n"),
uses: "google-github-actions/setup-gcloud@v0",
env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" },
env: {
CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe",
},
with: {
project_id: "denoland",
service_account_key: "${{ secrets.GCP_SA_KEY }}",
@ -193,68 +281,8 @@ const ci = {
run: 'echo "DENO_CANARY=true" >> $GITHUB_ENV',
},
{
name: "Set up incremental LTO and sysroot build",
if: "matrix.use_sysroot",
run: [
"# Avoid running man-db triggers, which sometimes takes several minutes",
"# to complete.",
"sudo apt-get remove --purge -y man-db",
"",
"# Install clang-15, lld-15, and debootstrap.",
'echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-15 main" |',
" sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-focal-15.list",
"curl https://apt.llvm.org/llvm-snapshot.gpg.key |",
" gpg --dearmor |",
"sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg",
"sudo apt-get update",
"sudo apt-get install --no-install-recommends debootstrap \\",
" clang-15 lld-15",
"",
"# Create ubuntu-16.04 sysroot environment, which is used to avoid",
"# depending on a very recent version of glibc.",
"# `libc6-dev` is required for building any C source files.",
"# `file` and `make` are needed to build libffi-sys.",
"# `curl` is needed to build rusty_v8.",
"sudo debootstrap \\",
" --include=ca-certificates,curl,file,libc6-dev,make \\",
" --no-merged-usr --variant=minbase xenial /sysroot \\",
" http://azure.archive.ubuntu.com/ubuntu",
"sudo mount --rbind /dev /sysroot/dev",
"sudo mount --rbind /sys /sysroot/sys",
"sudo mount --rbind /home /sysroot/home",
"sudo mount -t proc /proc /sysroot/proc",
"",
"# Configure the build environment. Both Rust and Clang will produce",
"# llvm bitcode only, so we can use lld's incremental LTO support.",
"cat >> $GITHUB_ENV << __0",
"CARGO_PROFILE_BENCH_INCREMENTAL=false",
"CARGO_PROFILE_BENCH_LTO=false",
"CARGO_PROFILE_RELEASE_INCREMENTAL=false",
"CARGO_PROFILE_RELEASE_LTO=false",
"RUSTFLAGS<<__1",
" -C linker-plugin-lto=true",
" -C linker=clang-15",
" -C link-arg=-fuse-ld=lld-15",
" -C link-arg=--sysroot=/sysroot",
" -C link-arg=-Wl,--allow-shlib-undefined",
" -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache",
" -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m",
" ${{ env.RUSTFLAGS }}",
"__1",
"RUSTDOCFLAGS<<__1",
" -C linker-plugin-lto=true",
" -C linker=clang-15",
" -C link-arg=-fuse-ld=lld-15",
" -C link-arg=--sysroot=/sysroot",
" -C link-arg=-Wl,--allow-shlib-undefined",
" -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache",
" -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m",
" ${{ env.RUSTFLAGS }}",
"__1",
"CC=clang-15",
"CFLAGS=-flto=thin --sysroot=/sysroot",
"__0",
].join("\n"),
...sysRootStep,
},
{
name: "Log versions",
@ -455,7 +483,9 @@ const ci = {
"github.repository == 'denoland/deno' &&",
"github.ref == 'refs/heads/main'",
].join("\n"),
env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" },
env: {
CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe",
},
shell: "bash",
run:
'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/',
@ -472,7 +502,9 @@ const ci = {
name: "Test fastci",
if: "(matrix.job == 'test' && matrix.profile == 'fastci')",
run: "cargo test --locked",
env: { CARGO_PROFILE_DEV_DEBUG: 0 },
env: {
CARGO_PROFILE_DEV_DEBUG: 0,
},
},
{
name: "Test release",
@ -492,7 +524,9 @@ const ci = {
"matrix.profile == 'release' && startsWith(github.ref, 'refs/tags/')",
shell: "bash",
run: 'target/release/deno eval "console.log(1+2)" | grep 3',
env: { NO_COLOR: 1 },
env: {
NO_COLOR: 1,
},
},
{
// Verify that the binary actually works in the Ubuntu-16.04 sysroot.
@ -515,7 +549,9 @@ const ci = {
"matrix.profile == 'debug' &&",
"github.ref == 'refs/heads/main'",
].join("\n"),
env: { DENO_BIN: "./target/debug/deno" },
env: {
DENO_BIN: "./target/debug/deno",
},
run: [
"deno run --allow-env --allow-net --allow-read --allow-run \\",
" --allow-write --unstable \\",
@ -533,7 +569,9 @@ const ci = {
"startsWith(matrix.os, 'ubuntu') && matrix.job == 'test' &&",
"matrix.profile == 'release' && !startsWith(github.ref, 'refs/tags/')",
].join("\n"),
env: { DENO_BIN: "./target/release/deno" },
env: {
DENO_BIN: "./target/release/deno",
},
run: [
"deno run --allow-env --allow-net --allow-read --allow-run \\",
" --allow-write --unstable \\",
@ -598,7 +636,9 @@ const ci = {
"github.repository == 'denoland/deno' &&",
"github.ref == 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/')",
].join("\n"),
env: { DENOBOT_PAT: "${{ secrets.DENOBOT_PAT }}" },
env: {
DENOBOT_PAT: "${{ secrets.DENOBOT_PAT }}",
},
run: [
"git clone --depth 1 --branch gh-pages \\",
" https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git \\",
@ -625,7 +665,10 @@ const ci = {
{
name: "Worker info",
if: "matrix.job == 'bench'",
run: ["cat /proc/cpuinfo", "cat /proc/meminfo"].join("\n"),
run: [
"cat /proc/cpuinfo",
"cat /proc/meminfo",
].join("\n"),
},
{
name: "Upload release to dl.deno.land (unix)",
@ -648,7 +691,9 @@ const ci = {
"github.repository == 'denoland/deno' &&",
"startsWith(github.ref, 'refs/tags/')",
].join("\n"),
env: { CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe" },
env: {
CLOUDSDK_PYTHON: "${{env.pythonLocation}}\\python.exe",
},
shell: "bash",
run:
'gsutil -h "Cache-Control: public, max-age=3600" cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/',
@ -676,7 +721,9 @@ const ci = {
"github.repository == 'denoland/deno' &&",
"startsWith(github.ref, 'refs/tags/')",
].join("\n"),
env: { GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" },
env: {
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}",
},
with: {
files: [
"target/release/deno-x86_64-pc-windows-msvc.zip",

26
.github/workflows/ci.yml vendored
View file

@ -58,7 +58,15 @@ jobs:
uses: actions/checkout@v3
with:
fetch-depth: 5
submodules: recursive
submodules: false
- name: Clone submodule ./test_util/std
run: git submodule update --init --recursive --depth=1 -- ./test_util/std
- name: Clone submodule ./test_util/wpt
run: git submodule update --init --recursive --depth=1 -- ./test_util/wpt
if: matrix.job == 'test'
- name: Clone submodule ./third_party
run: git submodule update --init --recursive --depth=1 -- ./third_party
if: matrix.job == 'lint' || matrix.job == 'bench'
- name: 'Create source tarballs (release, linux)'
if: |-
startsWith(matrix.os, 'ubuntu') &&
@ -71,8 +79,8 @@ jobs:
tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
-czvf target/release/deno_src.tar.gz -C .. deno
- uses: dtolnay/rust-toolchain@stable
- name: Install Deno
if: matrix.job == 'lint' || matrix.job == 'test'
- if: matrix.job == 'lint' || matrix.job == 'test'
name: Install Deno
uses: denoland/setup-deno@v1
with:
deno-version: v1.x
@ -80,10 +88,6 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: 3.8
- name: Install Node
uses: actions/setup-node@v3
with:
node-version: 17
- name: Remove unused versions of Python
if: 'startsWith(matrix.os, ''windows'')'
run: |-
@ -91,6 +95,10 @@ jobs:
Where-Object { Test-Path "$_\python.exe" } |
Select-Object -Skip 1 |
ForEach-Object { Move-Item "$_" "$_.disabled" }
- name: Install Node
uses: actions/setup-node@v3
with:
node-version: 17
- name: Setup gcloud (unix)
if: |-
runner.os != 'Windows' &&
@ -127,8 +135,8 @@ jobs:
github.ref == 'refs/heads/main'
shell: bash
run: echo "DENO_CANARY=true" >> $GITHUB_ENV
- name: Set up incremental LTO and sysroot build
if: matrix.use_sysroot
- if: matrix.use_sysroot
name: Set up incremental LTO and sysroot build
run: |-
# Avoid running man-db triggers, which sometimes takes several minutes
# to complete.