denoland-deno/.github/workflows/ci.yml

name: ci

on: [push, pull_request]

jobs:
  build:
    name: ${{ matrix.kind }} ${{ matrix.profile }} ${{ matrix.os }}
    if: |
      github.event_name == 'push' ||
      !startsWith(github.event.pull_request.head.label, 'denoland:')      
    runs-on: ${{ matrix.os }}
    timeout-minutes: 90
    strategy:
      matrix:
        include:
          # TODO(ry) Rename 'kind' to 'job'?
          - os: macos-10.15
            kind: test
            profile: release
          - os: windows-2019
            kind: test
            profile: release
          - os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
            kind: test
            profile: release
            use_sysroot: true
          - os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
            kind: bench
            profile: release
            use_sysroot: true
          - os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
            kind: test
            profile: debug
          - os: ${{ github.repository == 'denoland/deno' && 'ubuntu-latest-xl' || 'ubuntu-latest' }}
            kind: lint
            profile: debug

      # Always run main branch builds to completion. This allows the cache to
      # stay mostly up-to-date in situations where a single job fails due to
      # e.g. a flaky test.
      # Don't fast-fail on tag build because publishing binaries shouldn't be
      # prevented if if any of the stages fails (which can be a false negative).
      fail-fast: ${{ github.event_name == 'pull_request' ||
        (github.ref != 'refs/heads/main' &&
        !startsWith(github.ref, 'refs/tags/')) }}

    env:
      CARGO_TERM_COLOR: always
      RUST_BACKTRACE: full
      RUSTC_FORCE_INCREMENTAL: 1

    steps:
      - name: Configure git
        run: |
          git config --global core.symlinks true
          git config --global fetch.parallel 32          

      - name: Clone repository
        uses: actions/checkout@v2
        with:
          # Use depth > 1, because sometimes we need to rebuild main and if
          # other commits have landed it will become impossible to rebuild if
          # the checkout is too shallow.
          fetch-depth: 5
          submodules: recursive

      - name: Create source tarballs (release, linux)
        if: |
          startsWith(matrix.os, 'ubuntu') &&
          matrix.profile == 'release' &&
          matrix.kind == 'test' &&
          github.repository == 'denoland/deno' &&
          startsWith(github.ref, 'refs/tags/')          
        run: |
          mkdir -p target/release
          tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
              -czvf target/release/deno_src.tar.gz -C .. deno          

      - name: Install Rust
        uses: hecrj/setup-rust-action@v1
        with:
          rust-version: 1.53.0

      - name: Install clippy and rustfmt
        if: matrix.kind == 'lint'
        run: |
          rustup component add clippy
          rustup component add rustfmt          

      - name: Install Deno
        if: matrix.kind == 'lint'
        run: |
          curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.7.2
          echo "$HOME/.deno/bin" >> $GITHUB_PATH          

      - name: Install Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.8

      - name: Install Node
        uses: actions/setup-node@v2
        with:
          node-version: 16

      - name: Remove unused versions of Python
        if: startsWith(matrix.os, 'windows')
        run: |-
          $env:PATH -split ";" |
            Where-Object { Test-Path "$_\python.exe" } |
            Select-Object -Skip 1 |
            ForEach-Object { Move-Item "$_" "$_.disabled" }          

      - name: Setup gcloud (unix)
        if: |
          runner.os != 'Windows' &&
          matrix.profile == 'release' &&
          matrix.kind == 'test' &&
          github.repository == 'denoland/deno' &&
          (github.ref == 'refs/heads/main' ||
          startsWith(github.ref, 'refs/tags/'))          
        uses: google-github-actions/setup-gcloud@master
        with:
          project_id: denoland
          service_account_key: ${{ secrets.GCP_SA_KEY }}
          export_default_credentials: true

      - name: Setup gcloud (windows)
        if: |
          runner.os == 'Windows' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          (github.ref == 'refs/heads/main' ||
          startsWith(github.ref, 'refs/tags/'))          
        uses: google-github-actions/setup-gcloud@master
        env:
          CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
        with:
          project_id: denoland
          service_account_key: ${{ secrets.GCP_SA_KEY }}
          export_default_credentials: true

      - name: Error on warning
        # TODO(piscisaureus): enable this on Windows again.
        if: "!matrix.use_sysroot && !startsWith(matrix.os, 'windows')"
        run: echo "RUSTFLAGS=-D warnings" >> $GITHUB_ENV

      - name: Set up Linux sysroot with Ubuntu 18.04 and LLVM
        if: matrix.use_sysroot
        run: |
          sudo apt-get update
          sudo apt-get install debootstrap

          # Note: git, nc, strace, and time, are needed to run the benchmarks.
          sudo debootstrap                                                     \
                 --include=ca-certificates,curl,git,netcat-openbsd,strace,time \
                 --no-merged-usr --variant=minbase bionic /sysroot             \
                 http://azure.archive.ubuntu.com/ubuntu
          sudo mount --rbind /dev /sysroot/dev
          sudo mount --rbind /sys /sysroot/sys
          sudo mount --rbind /home /sysroot/home
          sudo mount -t proc /proc /sysroot/proc

          sudo ln --force --target /sysroot/etc \
                  /etc/passwd /etc/shadow /etc/group /etc/gshadow

          # Install clang-12 and lld-12 into the chroot environment.
          echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-12 main" |
            sudo dd of=/sysroot/etc/apt/sources.list.d/llvm-toolchain-bionic-12.list
          curl https://apt.llvm.org/llvm-snapshot.gpg.key |
            gpg --dearmor                                 |
            sudo dd of=/sysroot/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
          sudo chroot /sysroot apt update -y
          sudo chroot /sysroot apt install --no-install-recommends -y \
                                           clang-12 lld-12

          # Make rust available inside the chroot environment.
          sudo mkdir -p /sysroot/usr/share/rust
          sudo mount --rbind /usr/share/rust /sysroot/usr/share/rust

          # Make node (needed to run the benchmarks) available.
          sudo ln --target /sysroot/usr/bin "$(which node)"

          cat >> ~/.bash_profile << ___
            cd "$(pwd)"

            # Add cargo, rustc, and deno to $PATH.
            source /usr/share/rust/.cargo/env
            export PATH="$PATH:$(pwd)/target/release"

            # Rust build configuration.
            export CARGO_PROFILE_BENCH_INCREMENTAL=false
            export CARGO_PROFILE_BENCH_LTO=false
            export CARGO_PROFILE_RELEASE_INCREMENTAL=false
            export CARGO_PROFILE_RELEASE_LTO=false
            export RUSTFLAGS="
              -C linker-plugin-lto=true
              -C linker=clang-12
              -C link-arg=-fuse-ld=lld-12
              -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
              -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
              -D warnings
            "
            export RUSTDOCFLAGS="\$RUSTFLAGS"
            unset RUSTC_FORCE_INCREMENTAL

            # C build configuration.
            export CC=clang-12         # Compile c source files with clang.
            export CCFLAGs=-flto=thin  # Tell clang to produce llvm bitcode.

            # Miscellaneous flags.
            export CARGO_TERM_COLOR=always
            export CI=true
            export DENO_CANARY=true
          ___          

      - name: Log versions
        shell: bash
        run: |
          node -v
          python --version
          rustc --version
          cargo --version
          # Deno is installed when linting.
          if [ "${{ matrix.kind }}" == "lint" ]
          then
            deno --version
          fi          

      - name: Cache Cargo home
        uses: actions/cache@v2
        with:
          # See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
          path: |
            ~/.cargo/registry/index
            ~/.cargo/registry/cache
            ~/.cargo/git/db            
          key: z-cargo-home-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}

      # In main branch, always creates fresh cache
      - name: Cache build output (main)
        # TODO(kt3k): Change the version to the released version
        # when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        if: matrix.profile == 'release' && github.ref == 'refs/heads/main'
        with:
          path: |
            ./target
            !./target/*/gn_out
            !./target/*/*.zip
            !./target/*/*.tar.gz            
          key: |
            z-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-${{ github.sha }}            

      # Restore cache from the latest 'main' branch build.
      - name: Cache build output (PR)
        # TODO(kt3k): Change the version to the released version
        # when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        if: github.ref != 'refs/heads/main'
        with:
          path: |
            ./target
            !./target/*/gn_out
            !./target/*/*.zip
            !./target/*/*.tar.gz            
          key: never_saved
          restore-keys: |
            z-cargo-target-${{ matrix.os }}-${{ matrix.profile }}-            

      # Don't save cache after building PRs or branches other than 'main'.
      - name: Skip save cache (PR)
        run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
        shell: bash
        if: github.ref != 'refs/heads/main'

      - name: Apply and update mtime cache
        if: matrix.profile == 'release'
        uses: ./.github/mtime_cache
        with:
          cache-path: ./target

      # Shallow the cloning the crates.io index makes CI faster because it
      # obviates the need for Cargo to clone the index. If we don't do this
      # Cargo will `git clone` the github repository that contains the entire
      # history of the crates.io index from github. We don't believe the
      # identifier '1ecc6299db9ec823' will ever change, but if it does then this
      # command must be updated.
      - name: Shallow clone crates.io index
        shell: bash
        run: |
          if [ ! -d ~/.cargo/registry/index/github.com-1ecc6299db9ec823/.git ]
          then
            git clone --depth 1 --no-checkout                      \
                      https://github.com/rust-lang/crates.io-index \
                      ~/.cargo/registry/index/github.com-1ecc6299db9ec823
          fi          

      - name: Configure canary build
        if: |
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          !matrix.use_sysroot &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        shell: bash
        run: echo "DENO_CANARY=true" >> $GITHUB_ENV

      - name: test_format.js
        if: matrix.kind == 'lint'
        run: deno run --unstable --allow-write --allow-read --allow-run ./tools/format.js --check

      - name: lint.js
        if: matrix.kind == 'lint'
        # TODO(ry) assert matrix.profile == "debug"
        run: deno run --unstable --allow-write --allow-read --allow-run ./tools/lint.js

      - name: Build debug
        if: |
          (matrix.kind == 'test' || matrix.kind == 'bench') &&
          matrix.profile == 'debug' && !matrix.use_sysroot          
        run: cargo build --locked --all-targets

      - name: Build release
        if: |
          (matrix.kind == 'test' || matrix.kind == 'bench') &&
          matrix.profile == 'release' && !matrix.use_sysroot          
        run: cargo build --release --locked --all-targets

      - name: Build release (in sysroot)
        if: |
          (matrix.kind == 'test' || matrix.kind == 'bench') &&
          matrix.profile == 'release' && matrix.use_sysroot          
        run: |
          sudo chroot /sysroot   \
               su -l "$(whoami)" \
                  -c "cargo build --release --locked --all-targets"          

      - name: Pre-release (linux)
        if: |
          startsWith(matrix.os, 'ubuntu') &&
          matrix.kind == 'test' &&
          matrix.profile == 'release'          
        run: |
          cd target/release
          zip -r deno-x86_64-unknown-linux-gnu.zip deno
          ./deno types > lib.deno.d.ts          

      - name: Pre-release (mac)
        if: |
          startsWith(matrix.os, 'macOS') &&
          matrix.kind == 'test' &&
          matrix.profile == 'release'          
        run: |
          cd target/release
          zip -r deno-x86_64-apple-darwin.zip deno          

      - name: Pre-release (windows)
        if: |
          startsWith(matrix.os, 'windows') &&
          matrix.kind == 'test' &&
          matrix.profile == 'release'          
        run: |
          Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-x86_64-pc-windows-msvc.zip          

      - name: Upload canary to dl.deno.land (unix)
        if: |
          runner.os != 'Windows' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        run: |
          gsutil cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/
          echo $(git rev-parse HEAD) > canary-latest.txt
          gsutil cp canary-latest.txt gs://dl.deno.land/canary-latest.txt          

      - name: Upload canary to dl.deno.land (windows)
        if: |
          runner.os == 'Windows' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        env:
          CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
        shell: bash
        run: |
          gsutil cp ./target/release/*.zip gs://dl.deno.land/canary/$(git rev-parse HEAD)/
          echo $(git rev-parse HEAD) > canary-latest.txt
          gsutil cp canary-latest.txt gs://dl.deno.land/canary-latest.txt          

      - name: Test debug
        if: |
          matrix.kind == 'test' && matrix.profile == 'debug' &&
          !matrix.use_sysroot          
        run: |
          cargo test --locked --doc
          cargo test --locked          

      - name: Test release
        if: |
          matrix.kind == 'test' && matrix.profile == 'release' &&
          !matrix.use_sysroot          
        run: cargo test --release --locked

      - name: Test release (in sysroot)
        if: |
          matrix.kind == 'test' && matrix.profile == 'release' &&
          matrix.use_sysroot          
        run: |
          sudo chroot /sysroot   \
               su -l "$(whoami)" \
                  -c "cargo test --release --locked"          

      # TODO(ry): Because CI is so slow on for OSX and Windows, we currently
      #           run the Web Platform tests only on Linux.
      - name: Configure hosts file for WPT
        if: startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test'
        run: ./wpt make-hosts-file | sudo tee -a /etc/hosts
        working-directory: test_util/wpt/

      - name: Run web platform tests (debug)
        if: |
          startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' &&
          matrix.profile == 'debug'          
        env:
          DENO_BIN: ./target/debug/deno
        run: |
          "$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
                          --allow-write --unstable                         \
                          ./tools/wpt.ts setup
          "$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
                          --allow-write --unstable                         \
                          ./tools/wpt.ts run --quiet --binary="$DENO_BIN"          

      - name: Run web platform tests (release)
        if: |
          startsWith(matrix.os, 'ubuntu') && matrix.kind == 'test' &&
          matrix.profile == 'release'          
        env:
          DENO_BIN: ./target/release/deno
        run: |
          "$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
                          --allow-write --unstable                         \
                          ./tools/wpt.ts setup
          "$DENO_BIN" run --allow-env --allow-net --allow-read --allow-run \
                          --allow-write --unstable                         \
                          ./tools/wpt.ts run --quiet --release             \
                                             --binary="$DENO_BIN"          \
                                             --json=wpt.json               \
                                             --wptreport=wptreport.json          

      - name: Upload wpt results to dl.deno.land
        if: |
          runner.os == 'Linux' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        run: |
          gzip ./wptreport.json
          gsutil cp ./wpt.json gs://dl.deno.land/wpt/$(git rev-parse HEAD).json
          gsutil cp ./wptreport.json.gz gs://dl.deno.land/wpt/$(git rev-parse HEAD)-wptreport.json.gz
          echo $(git rev-parse HEAD) > wpt-latest.txt
          gsutil cp wpt-latest.txt gs://dl.deno.land/wpt-latest.txt          

      - name: Upload wpt results to wpt.fyi
        if: |
          runner.os == 'Linux' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        env:
          WPT_FYI_STAGING_USER: ${{ secrets.WPT_FYI_STAGING_USER }}
          WPT_FYI_STAGING_PW: ${{ secrets.WPT_FYI_STAGING_PW }}
          GITHUB_TOKEN: ${{ secrets.DENOBOT_PAT }}
        run: |
          ./target/release/deno run --allow-all \
              ./tools/upload_wptfyi.js $(git rev-parse HEAD) --ghstatus          

      - name: Run benchmarks
        if: matrix.kind == 'bench' && !matrix.use_sysroot
        run: cargo bench --locked

      - name: Run benchmarks (in sysroot)
        if: matrix.kind == 'bench' && matrix.use_sysroot
        run: |
          sudo chroot /sysroot   \
               su -l "$(whoami)" \
                  -c "cargo bench --locked"          

      - name: Post Benchmarks
        if: |
          matrix.kind == 'bench' &&
          github.repository == 'denoland/deno' &&
          github.ref == 'refs/heads/main'          
        env:
          DENOBOT_PAT: ${{ secrets.DENOBOT_PAT }}
        run: |
          git clone --depth 1 --branch gh-pages                             \
              https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git \
              gh-pages
          ./target/release/deno run --allow-all --unstable \
              ./tools/build_benchmark_jsons.js --release
          cd gh-pages
          git config user.email "propelml@gmail.com"
          git config user.name "denobot"
          git add .
          git commit --message "Update benchmarks"
          git push origin gh-pages          

      - name: Build product size info
        if: matrix.kind != 'lint'
        run: |
          du -hd1 "./target/${{ matrix.profile }}"
          du -ha  "./target/${{ matrix.profile }}/deno"          

      - name: Worker info
        if: matrix.kind == 'bench'
        run: |
          cat /proc/cpuinfo
          cat /proc/meminfo          

      - name: Upload release to dl.deno.land (unix)
        if: |
          runner.os != 'Windows' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          startsWith(github.ref, 'refs/tags/')          
        run: |
          gsutil cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/
          echo ${GITHUB_REF#refs/*/} > release-latest.txt
          gsutil cp release-latest.txt gs://dl.deno.land/release-latest.txt          

      - name: Upload release to dl.deno.land (windows)
        if: |
          runner.os == 'Windows' &&
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          startsWith(github.ref, 'refs/tags/')          
        env:
          CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
        shell: bash
        run: |
          gsutil cp ./target/release/*.zip gs://dl.deno.land/release/${GITHUB_REF#refs/*/}/
          echo ${GITHUB_REF#refs/*/} > release-latest.txt
          gsutil cp release-latest.txt gs://dl.deno.land/release-latest.txt          

      - name: Upload release to GitHub
        uses: softprops/action-gh-release@v1
        if: |
          matrix.kind == 'test' &&
          matrix.profile == 'release' &&
          github.repository == 'denoland/deno' &&
          startsWith(github.ref, 'refs/tags/')          
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          files: |
            target/release/deno-x86_64-pc-windows-msvc.zip
            target/release/deno-x86_64-unknown-linux-gnu.zip
            target/release/deno-x86_64-apple-darwin.zip
            target/release/deno_src.tar.gz
            target/release/lib.deno.d.ts            
          draft: true