0
0
Fork 0
mirror of https://github.com/denoland/deno.git synced 2024-10-29 08:58:01 -04:00
denoland-deno/.github/workflows/ci.yml
2021-07-21 18:39:20 +02:00

592 lines
22 KiB
YAML

name: ci
on: [push, pull_request]
jobs:
build:
name: ${{ matrix.kind }} ${{ matrix.profile }} ${{ matrix.os }}
if: |
github.event_name == 'push' ||
!startsWith(github.event.pull_request.head.label, 'denoland:')
runs-on: ${{ matrix.os }}
timeout-minutes: 90
strategy:
matrix:
include:
# TODO(ry) Rename 'kind' to 'job'?
- os: macos-10.15
kind: test
profile: release
- os: windows-2019
kind: test
profile: release
- os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
kind: test
profile: release
use_sysroot: true
- os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
kind: bench
profile: release
use_sysroot: true
- os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
kind: test
profile: debug
- os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
kind: lint
profile: debug
# Always run main branch builds to completion. This allows the cache to
# stay mostly up-to-date in situations where a single job fails due to
# e.g. a flaky test.
# Don't fast-fail on tag build because publishing binaries shouldn't be
# prevented if if any of the stages fails (which can be a false negative).
fail-fast: ${{ github.event_name == 'pull_request' ||
(github.ref != 'refs/heads/main' &&
!startsWith(github.ref, 'refs/tags/')) }}
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: full
RUSTC_FORCE_INCREMENTAL: 1
steps:
- name: Configure git
run: |
git config --global core.symlinks true
git config --global fetch.parallel 32
- name: Clone repository
uses: actions/checkout@v2
with:
# Use depth > 1, because sometimes we need to rebuild main and if
# other commits have landed it will become impossible to rebuild if
# the checkout is too shallow.
fetch-depth: 5
submodules: recursive
- name: Create source tarballs (release, linux)
if: |
startsWith(matrix.os, 'ubuntu') &&
matrix.profile == 'release' &&
matrix.kind == 'test' &&
github.repository == 'denoland/deno' &&
startsWith(github.ref, 'refs/tags/')
run: |
mkdir -p target/release
tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
-czvf target/release/deno_src.tar.gz -C .. deno
- name: Install Rust
uses: hecrj/setup-rust-action@v1
with:
rust-version: 1.53.0
- name: Install clippy and rustfmt
if: matrix.kind == 'lint'
run: |
rustup component add clippy
rustup component add rustfmt
- name: Install Deno
if: matrix.kind == 'lint'
run: |
curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.11.3
echo "$HOME/.deno/bin" >> $GITHUB_PATH
- name: Install Python
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install Node
uses: actions/setup-node@v2
with:
node-version: 16
- name: Remove unused versions of Python
if: startsWith(matrix.os, 'windows')
run: |-
$env:PATH -split ";" |
Where-Object { Test-Path "$_\python.exe" } |
Select-Object -Skip 1 |
ForEach-Object { Move-Item "$_" "$_.disabled" }
- name: Setup gcloud (unix)
if: |
runner.os != 'Windows' &&
matrix.profile == 'release' &&
matrix.kind == 'test' &&
github.repository == 'denoland/deno' &&
(github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/tags/'))
uses: google-github-actions/setup-gcloud@master
with:
project_id: denoland
service_account_key: ${{ secrets.GCP_SA_KEY }}
export_default_credentials: true
- name: Setup gcloud (windows)
if: |
runner.os == 'Windows' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
(github.ref == 'refs/heads/main' ||
startsWith(github.ref, 'refs/tags/'))
uses: google-github-actions/setup-gcloud@master
env:
CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
with:
project_id: denoland
service_account_key: ${{ secrets.GCP_SA_KEY }}
export_default_credentials: true
- name: Error on warning
# TODO(piscisaureus): enable this on Windows again.
if: "!matrix.use_sysroot && !startsWith(matrix.os, 'windows')"
run: echo "RUSTFLAGS=-D warnings" >> $GITHUB_ENV
- name: Configure canary build
if: |
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
shell: bash
run: echo "DENO_CANARY=true" >> $GITHUB_ENV
- name: Set up Linux sysroot with Ubuntu 18.04 and LLVM
if: matrix.use_sysroot
run: |
# Avoid running man-db triggers, which sometimes takes several minutes
# to complete.
sudo apt-get remove -y --purge man-db
sudo apt-get update
sudo apt-get install debootstrap
# Note: git, nc, strace, and time, are needed to run the benchmarks.
sudo debootstrap \
--include=ca-certificates,curl,git,netcat-openbsd,strace,time \
--no-merged-usr --variant=minbase bionic /sysroot \
http://azure.archive.ubuntu.com/ubuntu
sudo mount --rbind /dev /sysroot/dev
sudo mount --rbind /sys /sysroot/sys
sudo mount --rbind /home /sysroot/home
sudo mount -t proc /proc /sysroot/proc
sudo ln --force --target /sysroot/etc \
/etc/passwd /etc/shadow /etc/group /etc/gshadow
# Install clang-12 and lld-12 into the chroot environment.
echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" |
sudo dd of=/sysroot/etc/apt/sources.list.d/llvm-toolchain-bionic-12.list
curl https://apt.llvm.org/llvm-snapshot.gpg.key |
gpg --dearmor |
sudo dd of=/sysroot/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
sudo chroot /sysroot apt update -y
sudo chroot /sysroot apt install --no-install-recommends -y \
clang-12 lld-12
# Make rust available inside the chroot environment.
sudo mkdir -p /sysroot/usr/share/rust
sudo mount --rbind /usr/share/rust /sysroot/usr/share/rust
# Make node (needed to run the benchmarks) available.
sudo ln --target /sysroot/usr/bin "$(which node)"
cat >> ~/.bash_profile << ___
cd "$(pwd)"
# Configure canary build.
if [ "$DENO_CANARY" == "true" ]
then
export DENO_CANARY=true
fi
# Add cargo, rustc, and deno to $PATH.
source /usr/share/rust/.cargo/env
export PATH="$PATH:$(pwd)/target/release"
# Rust build configuration.
export CARGO_PROFILE_BENCH_INCREMENTAL=false
export CARGO_PROFILE_BENCH_LTO=false
export CARGO_PROFILE_RELEASE_INCREMENTAL=false
export CARGO_PROFILE_RELEASE_LTO=false
export RUSTFLAGS="
-C linker-plugin-lto=true
-C linker=clang-12
-C link-arg=-fuse-ld=lld-12
-C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
-C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
-D warnings
"
export RUSTDOCFLAGS="\$RUSTFLAGS"
unset RUSTC_FORCE_INCREMENTAL
# C build configuration.
export CC=clang-12 # Compile c source files with clang.
export CFLAGS=-flto=thin # Tell clang to produce llvm bitcode.
# Miscellaneous flags.
export CARGO_TERM_COLOR=always
export CI=true
___
- name: Log versions
shell: bash
run: |
node -v
python --version
rustc --version
cargo --version
# Deno is installed when linting.
if [ "${{ matrix.kind }}" == "lint" ]
then
deno --version
fi
- name: Cache Cargo home
uses: actions/cache@v2
with:
# See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
path: |
~/.cargo/registry/index
~/.cargo/registry/cache
~/.cargo/git/db
key: e-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
# In main branch, always creates fresh cache
- name: Cache build output (main)
# TODO(kt3k): Change the version to the released version
# when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
if: matrix.profile == 'release' && github.ref == 'refs/heads/main'
with:
path: |
./target
!./target/*/gn_out
!./target/*/*.zip
!./target/*/*.tar.gz
key: |
d-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}
# Restore cache from the latest 'main' branch build.
- name: Cache build output (PR)
# TODO(kt3k): Change the version to the released version
# when https://github.com/actions/cache/pull/489 (or 571) is merged.
uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
if: github.ref != 'refs/heads/main'
with:
path: |
./target
!./target/*/gn_out
!./target/*/*.zip
!./target/*/*.tar.gz
key: never_saved
restore-keys: |
d-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-
# Don't save cache after building PRs or branches other than 'main'.
- name: Skip save cache (PR)
run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
shell: bash
if: github.ref != 'refs/heads/main'
- name: Apply and update mtime cache
if: matrix.profile == 'release'
uses: ./.github/mtime_cache
with:
cache-path: ./target
# Shallow the cloning the crates.io index makes CI faster because it
# obviates the need for Cargo to clone the index. If we don't do this
# Cargo will `git clone` the github repository that contains the entire
# history of the crates.io index from github. We don't believe the
# identifier '1ecc6299db9ec823' will ever change, but if it does then this
# command must be updated.
- name: Shallow clone crates.io index
shell: bash
run: |
if [ ! -d ~/.cargo/registry/index/github.com-1ecc6299db9ec823/.git ]
then
git clone --depth 1 --no-checkout \
https://github.com/rust-lang/crates.io-index \
~/.cargo/registry/index/github.com-1ecc6299db9ec823
fi
- name: test_format.js
if: matrix.kind == 'lint'
run: deno run --unstable --allow-write --allow-read --allow-run ./tools/format.js --check
- name: lint.js
if: matrix.kind == 'lint'
# TODO(ry) assert matrix.profile == "debug"
run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js
- name: Build debug
if: |
(matrix.kind == 'test' || matrix.kind == 'bench') &&
matrix.profile == 'debug' && !matrix.use_sysroot
run: cargo build --locked --all-targets
- name: Build release
if: |
(matrix.kind == 'test' || matrix.kind == 'bench') &&
matrix.profile == 'release' && !matrix.use_sysroot
run: cargo build --release --locked --all-targets
- name: Build release (in sysroot)
if: |
(matrix.kind == 'test' || matrix.kind == 'bench') &&
matrix.profile == 'release' && matrix.use_sysroot
run: |
sudo chroot /sysroot \
su -l "$(whoami)" \
-c "cargo build --release --locked --all-targets"
- name: Pre-release (linux)
if: |
startsWith(matrix.os, 'ubuntu') &&
matrix.kind == 'test' &&
matrix.profile == 'release'
run: |
cd target/release
zip -r deno-x86_64-unknown-linux-gnu.zip deno
./deno types > lib.deno.d.ts
- name: Pre-release (mac)
if: |
startsWith(matrix.os, 'macOS') &&
matrix.kind == 'test' &&
matrix.profile == 'release'
run: |
cd target/release
zip -r deno-x86_64-apple-darwin.zip deno
- name: Pre-release (windows)
if: |
startsWith(matrix.os, 'windows') &&
matrix.kind == 'test' &&
matrix.profile == 'release'
run: |
Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-x86_64-pc-windows-msvc.zip
- name: Upload canary to dl.deno.land (unix)
if: |
runner.os != 'Windows' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
run: |
gsutil cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/
- name: Upload canary to dl.deno.land (windows)
if: |
runner.os == 'Windows' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
env:
CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
shell: bash
run: |
gsutil cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/
- name: Test debug
if: |
matrix.kind == 'test' && matrix.profile == 'debug' &&
!matrix.use_sysroot
run: |
cargo test --locked --doc
cargo test --locked
- name: Test release
if: |
matrix.kind == 'test' && matrix.profile == 'release' &&
!matrix.use_sysroot
run: cargo test --release --locked
- name: Test release (in sysroot)
if: |
matrix.kind == 'test' && matrix.profile == 'release' &&
matrix.use_sysroot
run: |
sudo chroot /sysroot \
su -l "$(whoami)" \
-c "cargo test --release --locked"
# TODO(ry): Because CI is so slow on for OSX and Windows, we currently
# run the Web Platform tests only on Linux.
- name: Configure hosts file for WPT
if: startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test'
run: ./wpt make-hosts-file | sudo tee -a /etc/hosts
working-directory: test_util/wpt/
- name: Run web platform tests (debug)
if: |
startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' &&
matrix.profile == 'debug'
env:
DENO_BIN: ./target/debug/deno
run: |
"$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
--allow-write --unstable \
./tools/wpt.ts setup
"$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
--allow-write --unstable \
./tools/wpt.ts run --quiet --binary="$DENO_BIN"
- name: Run web platform tests (release)
if: |
startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' &&
matrix.profile == 'release'
env:
DENO_BIN: ./target/release/deno
run: |
"$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
--allow-write --unstable \
./tools/wpt.ts setup
"$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
--allow-write --unstable \
./tools/wpt.ts run --quiet --release \
--binary="$DENO_BIN" \
--json=wpt.json \
--wptreport=wptreport.json
- name: Upload wpt results to dl.deno.land
if: |
runner.os == 'Linux' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
run: |
gzip ./wptreport.json
gsutil cp ./wpt.json gs://dl.deno.land/wpt/$(git rev-parse HEAD).json
gsutil cp ./wptreport.json.gz gs://dl.deno.land/wpt/$(git rev-parse HEAD)-wptreport.json.gz
echo $(git rev-parse HEAD) > wpt-latest.txt
gsutil cp wpt-latest.txt gs://dl.deno.land/wpt-latest.txt
- name: Upload wpt results to wpt.fyi
if: |
runner.os == 'Linux' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
env:
WPT_FYI_USER: ${{ secrets.WPT_FYI_USER }}
WPT_FYI_PW: ${{ secrets.WPT_FYI_PW }}
GITHUB_TOKEN: ${{ secrets.DENOBOT_PAT }}
run: |
./target/release/deno run --allow-all \
./tools/upload_wptfyi.js $(git rev-parse HEAD) --ghstatus
- name: Run benchmarks
if: matrix.kind == 'bench' && !matrix.use_sysroot
run: cargo bench --locked
- name: Run benchmarks (in sysroot)
if: matrix.kind == 'bench' && matrix.use_sysroot
run: |
sudo chroot /sysroot \
su -l "$(whoami)" \
-c "cargo bench --locked"
- name: Post Benchmarks
if: |
matrix.kind == 'bench' &&
github.repository == 'denoland/deno' &&
github.ref == 'refs/heads/main'
env:
DENOBOT_PAT: ${{ secrets.DENOBOT_PAT }}
run: |
git clone --depth 1 --branch gh-pages \
https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git \
gh-pages
./target/release/deno run --allow-all --unstable \
./tools/build_benchmark_jsons.js --release
cd gh-pages
git config user.email "propelml@gmail.com"
git config user.name "denobot"
git add .
git commit --message "Update benchmarks"
git push origin gh-pages
- name: Build product size info
if: matrix.kind != 'lint'
run: |
du -hd1 "./target/${{ matrix.profile }}"
du -ha "./target/${{ matrix.profile }}/deno"
- name: Worker info
if: matrix.kind == 'bench'
run: |
cat /proc/cpuinfo
cat /proc/meminfo
- name: Upload release to dl.deno.land (unix)
if: |
runner.os != 'Windows' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
startsWith(github.ref, 'refs/tags/')
run: |
gsutil cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/
echo ${GITHUB_REF#refs/*/} > release-latest.txt
gsutil cp release-latest.txt gs://dl.deno.land/release-latest.txt
- name: Upload release to dl.deno.land (windows)
if: |
runner.os == 'Windows' &&
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
startsWith(github.ref, 'refs/tags/')
env:
CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
shell: bash
run: |
gsutil cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/
echo ${GITHUB_REF#refs/*/} > release-latest.txt
gsutil cp release-latest.txt gs://dl.deno.land/release-latest.txt
- name: Upload release to GitHub
uses: softprops/action-gh-release@v1
if: |
matrix.kind == 'test' &&
matrix.profile == 'release' &&
github.repository == 'denoland/deno' &&
startsWith(github.ref, 'refs/tags/')
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
files: |
target/release/deno-x86_64-pc-windows-msvc.zip
target/release/deno-x86_64-unknown-linux-gnu.zip
target/release/deno-x86_64-apple-darwin.zip
target/release/deno_src.tar.gz
target/release/lib.deno.d.ts
draft: true
publish-canary:
name: publish canary
runs-on: ubuntu-20.04
needs: ['build']
if: github.repository == 'denoland/deno' &&
(github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
steps:
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@master
with:
project_id: denoland
service_account_key: ${{ secrets.GCP_SA_KEY }}
export_default_credentials: true
- name: Upload canary version file to dl.deno.land
run: |
echo ${{ github.sha }} > canary-latest.txt
gsutil cp canary-latest.txt gs://dl.deno.land/canary-latest.txt