ci.yml

# GENERATED BY ./ci.generate.ts -- DO NOT DIRECTLY EDIT

name: ci
permissions:
  contents: write
on:
  push:
    branches:
      - main
    tags:
      - '*'
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
concurrency:
  group: '${{ github.workflow }}-${{ !contains(github.event.pull_request.labels.*.name, ''ci-test-flaky'') && github.head_ref || github.run_id }}'
  cancel-in-progress: true
jobs:
  pre_build:
    name: pre-build
    runs-on: ubuntu-latest
    outputs:
      skip_build: '${{ steps.check.outputs.skip_build }}'
    steps:
      - name: Configure git
        run: |-
          git config --global core.symlinks true
          git config --global fetch.parallel 32
        if: github.event.pull_request.draft == true
      - name: Clone repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 5
          submodules: false
        if: github.event.pull_request.draft == true
      - id: check
        if: 'github.event.pull_request.draft == true && (!contains(github.event.pull_request.labels.*.name, ''ci-draft''))'
        run: |-
          GIT_MESSAGE=$(git log --format=%s -n 1 ${{github.event.after}})
          echo Commit message: $GIT_MESSAGE
          echo $GIT_MESSAGE | grep '\[ci\]' || (echo 'Exiting due to draft PR. Commit with [ci] to bypass or add the ci-draft label.' ; echo 'skip_build=true' >> $GITHUB_OUTPUT)
  build:
    name: '${{ matrix.job }} ${{ matrix.profile }} ${{ matrix.os }}-${{ matrix.arch }}'
    needs:
      - pre_build
    if: '${{ needs.pre_build.outputs.skip_build != ''true'' }}'
    runs-on: '${{ matrix.runner }}'
    timeout-minutes: 150
    defaults:
      run:
        shell: bash
    strategy:
      matrix:
        include:
          - os: macos
            arch: x86_64
            runner: macos-13
            job: build
            profile: debug
          - os: macos
            arch: x86_64
            runner: macos-13
            job: build
            profile: release
          - os: macos
            arch: aarch64
            runner: macos-14
            job: build
            profile: debug
          - os: macos
            arch: aarch64
            runner: macos-14
            job: build
            profile: release
          - os: windows
            arch: x86_64
            runner: windows-2022
            job: build
            profile: debug
          - os: windows
            arch: x86_64
            runner: windows-2022
            job: build
            profile: release
          - os: linux
            arch: x86_64
            runner: ubuntu-22.04
            job: build
            profile: release
            use_sysroot: true
            wpt: '${{ !startsWith(github.ref, ''refs/tags/'') }}'
          - os: linux
            arch: x86_64
            runner: ubuntu-22.04
            job: build
            profile: debug
            use_sysroot: true
      fail-fast: '${{ github.event_name == ''pull_request'' || (github.ref != ''refs/heads/main'' && !startsWith(github.ref, ''refs/tags/'')) }}'
    env:
      CARGO_TERM_COLOR: always
      RUST_BACKTRACE: full
      RUST_LIB_BACKTRACE: 0
    steps:
      - name: Configure git
        run: |-
          git config --global core.symlinks true
          git config --global fetch.parallel 32
        if: '!(matrix.skip)'
      - name: Clone repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 5
          submodules: false
        if: '!(matrix.skip)'
      - name: Clone submodule ./tests/util/std
        run: git submodule update --init --recursive --depth=1 -- ./tests/util/std
        if: '!(matrix.skip)'
      - name: Clone submodule ./tests/wpt/suite
        run: git submodule update --init --recursive --depth=1 -- ./tests/wpt/suite
        if: '!(matrix.skip) && (matrix.wpt)'
      - name: Clone submodule ./tests/node_compat/runner/suite
        run: git submodule update --init --recursive --depth=1 -- ./tests/node_compat/runner/suite
        if: '!(matrix.skip) && (matrix.job == ''lint'' && matrix.os == ''linux'')'
      - name: Clone submodule ./cli/bench/testdata/lsp_benchdata
        run: git submodule update --init --recursive --depth=1 -- ./cli/bench/testdata/lsp_benchdata
        if: '!(matrix.skip) && (matrix.job == ''bench'')'
      - name: 'Create source tarballs (release, linux)'
        if: |-
          !(matrix.skip) && (matrix.os == 'linux' &&
          matrix.profile == 'release' &&
          matrix.job == 'build' &&
          github.repository == 'unyt-org/deno' &&
          startsWith(github.ref, 'refs/tags/'))
        run: |-
          mkdir -p target/release
          tar --exclude=".git*" --exclude=target --exclude=third_party/prebuilt \
              -czvf target/release/deno_src.tar.gz -C .. deno
      - uses: dsherret/rust-toolchain-file@v1
        if: '!(matrix.skip)'
      - if: '!(matrix.skip) && (matrix.job == ''lint'' || matrix.job == ''build'' || matrix.job == ''bench'')'
        name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
        if: '!(matrix.skip) && (matrix.job != ''lint'' && (matrix.os != ''linux'' || matrix.arch != ''aarch64''))'
      - name: Remove unused versions of Python
        if: '!(matrix.skip) && (matrix.job != ''lint'' && (matrix.os != ''linux'' || matrix.arch != ''aarch64'') && (matrix.os == ''windows''))'
        shell: pwsh
        run: |-
          $env:PATH -split ";" |
            Where-Object { Test-Path "$_\python.exe" } |
            Select-Object -Skip 1 |
            ForEach-Object { Move-Item "$_" "$_.disabled" }
      - if: '!(matrix.skip) && (matrix.job == ''bench'' || matrix.job == ''build'')'
        name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install protoc
        uses: arduino/setup-protoc@v3
        with:
          version: '21.12'
          repo-token: '${{ secrets.GITHUB_TOKEN }}'
        if: '!(matrix.skip)'
      - name: Configure canary build
        if: |-
          !(matrix.skip) && (matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno' &&
          github.ref == 'refs/heads/main')
        run: echo "DENO_CANARY=true" >> $GITHUB_ENV
      - if: '!(matrix.skip) && (matrix.use_sysroot)'
        name: Set up incremental LTO and sysroot build
        run: |-
          # Setting up sysroot
          export DEBIAN_FRONTEND=noninteractive
          # Avoid running man-db triggers, which sometimes takes several minutes
          # to complete.
          sudo apt-get -qq remove --purge -y man-db  > /dev/null 2> /dev/null
          # Remove older clang before we install
          sudo apt-get -qq remove   'clang-12*' 'clang-13*' 'clang-14*' 'clang-15*' 'clang-16*' 'llvm-12*' 'llvm-13*' 'llvm-14*' 'llvm-15*' 'llvm-16*' 'lld-12*' 'lld-13*' 'lld-14*' 'lld-15*' 'lld-16*' > /dev/null 2> /dev/null

          # Install clang-XXX, lld-XXX, and debootstrap.
          echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-18 main" |
            sudo dd of=/etc/apt/sources.list.d/llvm-toolchain-jammy-18.list
          curl https://apt.llvm.org/llvm-snapshot.gpg.key |
            gpg --dearmor                                 |
          sudo dd of=/etc/apt/trusted.gpg.d/llvm-snapshot.gpg
          sudo apt-get update
          # this was unreliable sometimes, so try again if it fails
          sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18 || echo 'Failed. Trying again.' && sudo apt-get clean && sudo apt-get update && sudo apt-get install --no-install-recommends clang-18 lld-18 clang-tools-18 clang-format-18 clang-tidy-18
          # Fix alternatives
          (yes '' | sudo update-alternatives --force --all) > /dev/null 2> /dev/null || true

          echo "Decompressing sysroot..."
          wget -q https://github.com/denoland/deno_sysroot_build/releases/download/sysroot-20240528/sysroot-`uname -m`.tar.xz -O /tmp/sysroot.tar.xz
          cd /
          xzcat /tmp/sysroot.tar.xz | sudo tar -x
          sudo mount --rbind /dev /sysroot/dev
          sudo mount --rbind /sys /sysroot/sys
          sudo mount --rbind /home /sysroot/home
          sudo mount -t proc /proc /sysroot/proc
          cd

          echo "Done."

          # Configure the build environment. Both Rust and Clang will produce
          # llvm bitcode only, so we can use lld's incremental LTO support.

          # Load the sysroot's env vars
          echo "sysroot env:"
          cat /sysroot/.env
          . /sysroot/.env

          # Important notes:
          #   1. -ldl seems to be required to avoid a failure in FFI tests. This flag seems
          #      to be in the Rust default flags in the smoketest, so uncertain why we need
          #      to be explicit here.
          #   2. RUSTFLAGS and RUSTDOCFLAGS must be specified, otherwise the doctests fail
          #      to build because the object formats are not compatible.
          echo "
          CARGO_PROFILE_BENCH_INCREMENTAL=false
          CARGO_PROFILE_BENCH_LTO=false
          CARGO_PROFILE_RELEASE_INCREMENTAL=false
          CARGO_PROFILE_RELEASE_LTO=false
          RUSTFLAGS<<__1
            -C linker-plugin-lto=true
            -C linker=clang-18
            -C link-arg=-fuse-ld=lld-18
            -C link-arg=-ldl
            -C link-arg=-Wl,--allow-shlib-undefined
            -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
            -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
            --cfg tokio_unstable
            $RUSTFLAGS
          __1
          RUSTDOCFLAGS<<__1
            -C linker-plugin-lto=true
            -C linker=clang-18
            -C link-arg=-fuse-ld=lld-18
            -C link-arg=-ldl
            -C link-arg=-Wl,--allow-shlib-undefined
            -C link-arg=-Wl,--thinlto-cache-dir=$(pwd)/target/release/lto-cache
            -C link-arg=-Wl,--thinlto-cache-policy,cache_size_bytes=700m
            --cfg tokio_unstable
            $RUSTFLAGS
          __1
          CC=/usr/bin/clang-18
          CFLAGS=-flto=thin $CFLAGS
          " > $GITHUB_ENV
      - name: Remove macOS cURL --ipv4 flag
        run: |-
          curl --version
          which curl
          cat /etc/hosts
          rm ~/.curlrc || true
        if: '!(matrix.skip) && (matrix.os == ''macos'')'
      - name: Install macOS aarch64 lld
        run: ./tools/install_prebuilt.js ld64.lld
        if: '!(matrix.skip) && (matrix.os == ''macos'' && matrix.arch == ''aarch64'')'
      - name: Install rust-codesign
        run: |-
          ./tools/install_prebuilt.js rcodesign
          echo $GITHUB_WORKSPACE/third_party/prebuilt/mac >> $GITHUB_PATH
        if: '!(matrix.skip) && (matrix.os == ''macos'')'
      - name: Log versions
        run: |-
          echo '*** Python'
          command -v python && python --version || echo 'No python found or bad executable'
          echo '*** Rust'
          command -v rustc && rustc --version || echo 'No rustc found or bad executable'
          echo '*** Cargo'
          command -v cargo && cargo --version || echo 'No cargo found or bad executable'
          echo '*** Deno'
          command -v deno && deno --version || echo 'No deno found or bad executable'
          echo '*** Node'
          command -v node && node --version || echo 'No node found or bad executable'
          echo '*** Installed packages'
          command -v dpkg && dpkg -l || echo 'No dpkg found or bad executable'
        if: '!(matrix.skip)'
      - name: Install benchmark tools
        if: '!(matrix.skip) && (matrix.job == ''bench'')'
        run: ./tools/install_prebuilt.js wrk hyperfine
      - name: Cache Cargo home
        uses: actions/cache@v4
        with:
          path: |-
            ~/.cargo/registry/index
            ~/.cargo/registry/cache
          key: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}-${{ hashFiles(''Cargo.lock'') }}'
          restore-keys: '22-cargo-home-${{ matrix.os }}-${{ matrix.arch }}'
        if: '!(matrix.skip)'
      - name: Restore cache build output (PR)
        uses: actions/cache/restore@v4
        if: '!(matrix.skip) && (github.ref != ''refs/heads/main'' && !startsWith(github.ref, ''refs/tags/''))'
        with:
          path: |-
            ./target
            !./target/*/gn_out
            !./target/*/gn_root
            !./target/*/*.zip
            !./target/*/*.tar.gz
          key: never_saved
          restore-keys: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-'
      - name: Apply and update mtime cache
        if: '!(matrix.skip) && (!startsWith(github.ref, ''refs/tags/''))'
        uses: ./.github/mtime_cache
        with:
          cache-path: ./target
      - name: test_format.js
        if: '!(matrix.skip) && (matrix.job == ''lint'' && matrix.os == ''linux'')'
        run: deno run --allow-write --allow-read --allow-run --allow-net ./tools/format.js --check
      - name: Lint PR title
        if: '!(matrix.skip) && (matrix.job == ''lint'' && github.event_name == ''pull_request'' && matrix.os == ''linux'')'
        env:
          PR_TITLE: '${{ github.event.pull_request.title }}'
        run: deno run ./tools/verify_pr_title.js "$PR_TITLE"
      - name: lint.js
        if: '!(matrix.skip) && (matrix.job == ''lint'')'
        run: deno run --allow-write --allow-read --allow-run --allow-net ./tools/lint.js
      - name: jsdoc_checker.js
        if: '!(matrix.skip) && (matrix.job == ''lint'')'
        run: deno run --allow-read --allow-env --allow-sys ./tools/jsdoc_checker.js
      - name: node_compat/setup.ts --check
        if: '!(matrix.skip) && (matrix.job == ''lint'' && matrix.os == ''linux'')'
        run: deno run --allow-write --allow-read --allow-run=git ./tests/node_compat/runner/setup.ts --check
      - name: Build debug
        if: '!(matrix.skip) && (matrix.job == ''build'' && matrix.profile == ''debug'')'
        run: |-
          df -h
          cargo build --locked --all-targets
          df -h
        env:
          CARGO_PROFILE_DEV_DEBUG: 0
      - name: Build release
        if: |-
          !(matrix.skip) && ((matrix.job == 'build' || matrix.job == 'bench') &&
          matrix.profile == 'release' && (matrix.use_sysroot ||
          github.repository == 'unyt-org/deno'))
        run: |-
          df -h
          cargo build --release --locked --all-targets
          df -h
      - name: Check deno binary
        if: '!(matrix.skip) && (matrix.job == ''build'')'
        run: 'target/${{ matrix.profile }}/deno eval "console.log(1+2)" | grep 3'
        env:
          NO_COLOR: 1
      - name: Check deno binary (in sysroot)
        if: '!(matrix.skip) && (matrix.job == ''build'' && matrix.use_sysroot)'
        run: 'sudo chroot /sysroot "$(pwd)/target/${{ matrix.profile }}/deno" --version'
      - name: Upload PR artifact (GitHub)
        if: |-
          !(matrix.skip) && (matrix.job == 'build' &&
          (matrix.profile == 'release' || matrix.profile == 'debug') && (matrix.use_sysroot ||
          (github.repository == 'unyt-org/deno' &&
          (github.ref == 'refs/heads/main' ||
          startsWith(github.ref, 'refs/tags/')))))
        uses: actions/upload-artifact@v4
        with:
          name: 'deno-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}'
          path: 'target/${{ matrix.profile }}/deno'
      - name: Upload Debug artifact to dl.unyt.land
        if: |-
          !(matrix.skip) && (${{ github.event_name == 'pull_request' }}
          matrix.job == 'build' &&
          (matrix.profile == 'debug') && (matrix.use_sysroot ||
          (github.repository == 'unyt-org/deno' &&
          (github.ref == 'refs/heads/main' ||
          startsWith(github.ref, 'refs/tags/')))))
        run: |-
          echo 'Artifact URL is ${{ steps.artifact-upload-step.outputs.artifact-url }}'
          echo 'Artifact version is ${{ github.event.pull_request.title }}'
      - name: Pre-release (linux)
        if: |-
          !(matrix.skip) && (matrix.os == 'linux' &&
          matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno')
        run: |-
          cd target/release
          zip -r deno-${{ matrix.arch }}-unknown-linux-gnu.zip deno
          strip denort
          zip -r denort-${{ matrix.arch }}-unknown-linux-gnu.zip denort
          ./deno types > lib.deno.d.ts
      - name: Pre-release (mac)
        if: |-
          !(matrix.skip) && (matrix.os == 'macos' &&
          matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno')
        env:
          APPLE_CODESIGN_KEY: '${{ secrets.APPLE_CODESIGN_KEY }}'
          APPLE_CODESIGN_PASSWORD: '${{ secrets.APPLE_CODESIGN_PASSWORD }}'
        run: |-
          echo "Key is $(echo $APPLE_CODESIGN_KEY | base64 -d | wc -c) bytes"
          echo "Password is $(echo $APPLE_CODESIGN_PASSWORD | base64 -d | wc -c) bytes"
          rcodesign sign target/release/deno --code-signature-flags=runtime --p12-password="$APPLE_CODESIGN_PASSWORD" --p12-file=<(echo $APPLE_CODESIGN_KEY | base64 -d) --entitlements-xml-file=cli/entitlements.plist
          cd target/release
          zip -r deno-${{ matrix.arch }}-apple-darwin.zip deno
          strip denort
          zip -r denort-${{ matrix.arch }}-apple-darwin.zip denort
      - name: Pre-release (windows)
        if: |-
          !(matrix.skip) && (matrix.os == 'windows' &&
          matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno')
        shell: pwsh
        run: |-
          Compress-Archive -CompressionLevel Optimal -Force -Path target/release/deno.exe -DestinationPath target/release/deno-${{ matrix.arch }}-pc-windows-msvc.zip
          Compress-Archive -CompressionLevel Optimal -Force -Path target/release/denort.exe -DestinationPath target/release/denort-${{ matrix.arch }}-pc-windows-msvc.zip
      - name: Run benchmarks
        if: '!(matrix.skip) && (matrix.job == ''bench'' && !startsWith(github.ref, ''refs/tags/''))'
        run: cargo bench --locked
      - name: Post Benchmarks
        if: |-
          !(matrix.skip) && (matrix.job == 'bench' &&
          github.repository == 'unyt-org/deno' &&
          github.ref == 'refs/heads/main' && !startsWith(github.ref, 'refs/tags/'))
        env:
          DENOBOT_PAT: '${{ secrets.DENOBOT_PAT }}'
        run: |-
          git clone --depth 1 --branch gh-pages                             \
              https://${DENOBOT_PAT}@github.com/denoland/benchmark_data.git \
              gh-pages
          ./target/release/deno run --allow-all ./tools/build_benchmark_jsons.js --release
          cd gh-pages
          git config user.email "propelml@gmail.com"
          git config user.name "denobot"
          git add .
          git commit --message "Update benchmarks"
          git push origin gh-pages
      - name: Build product size info
        if: '!(matrix.skip) && (matrix.job != ''lint'' && matrix.profile != ''debug'' && github.repository == ''unyt-org/deno'' && (github.ref == ''refs/heads/main'' || startsWith(github.ref, ''refs/tags/'')))'
        run: |-
          du -hd1 "./target/${{ matrix.profile }}"
          du -ha  "./target/${{ matrix.profile }}/deno"
          du -ha  "./target/${{ matrix.profile }}/denort"
      - name: Worker info
        if: '!(matrix.skip) && (matrix.job == ''bench'')'
        run: |-
          cat /proc/cpuinfo
          cat /proc/meminfo
      - name: Create release notes
        if: |-
          !(matrix.skip) && (matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno' &&
          startsWith(github.ref, 'refs/tags/'))
        run: |-
          export PATH=$PATH:$(pwd)/target/release
          ./tools/release/05_create_release_notes.ts
      - name: Upload release to GitHub
        uses: softprops/action-gh-release@v0.1.15
        if: |-
          !(matrix.skip) && (matrix.job == 'build' &&
          matrix.profile == 'release' &&
          github.repository == 'unyt-org/deno' &&
          startsWith(github.ref, 'refs/tags/'))
        env:
          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
        with:
          files: |-
            target/release/deno-x86_64-pc-windows-msvc.zip
            target/release/denort-x86_64-pc-windows-msvc.zip
            target/release/deno-x86_64-unknown-linux-gnu.zip
            target/release/denort-x86_64-unknown-linux-gnu.zip
            target/release/deno-x86_64-apple-darwin.zip
            target/release/denort-x86_64-apple-darwin.zip
            target/release/deno-aarch64-unknown-linux-gnu.zip
            target/release/denort-aarch64-unknown-linux-gnu.zip
            target/release/deno-aarch64-apple-darwin.zip
            target/release/denort-aarch64-apple-darwin.zip
            target/release/deno_src.tar.gz
            target/release/lib.deno.d.ts
          body_path: target/release/release-notes.md
          draft: true
      - name: Save cache build output (main)
        uses: actions/cache/save@v4
        if: '!(matrix.skip) && ((matrix.job == ''build'' || matrix.job == ''lint'') && github.ref == ''refs/heads/main'')'
        with:
          path: |-
            ./target
            !./target/*/gn_out
            !./target/*/*.zip
            !./target/*/*.tar.gz
          key: '22-cargo-target-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.profile }}-${{ matrix.job }}-${{ github.sha }}'
  post-build:
    name: post-build
    runs-on: ubuntu-22.04
    needs:
      - build
    if: github.repository == 'unyt-org/deno' && github.ref == 'refs/heads/main'
    steps:
      - name: Notify dl.unyt.org
        run: 'curl ''https://dl.unyt.land/admin/update/${{ secrets.DL_UNYT_TOKEN }}'''