ci.yml

# Copyright 2022 The Fuchsia Authors
#
# Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
# <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
# license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.

name: Build & Tests

on:
  pull_request:
  push:
    branches:
      - main
      - v0.6.x
  merge_group:

permissions: read-all

env:
  CARGO_TERM_COLOR: always
  RUSTFLAGS: -Dwarnings
  RUSTDOCFLAGS: -Dwarnings
  # `ZC_NIGHTLY_XXX` are flags that we add to `XXX` only on the nightly
  # toolchain.
  ZC_NIGHTLY_RUSTFLAGS: -Zrandomize-layout
  ZC_NIGHTLY_MIRIFLAGS: "-Zmiri-strict-provenance -Zmiri-backtrace=full"

jobs:
  build_test:
    runs-on: ubuntu-latest
    # Generate and populate the global Cargo registry and cache first. Each
    # job in the matrix runs in parallel, so without populating the cache
    # first, most jobs would duplicate the work of downloading crates from
    # the internet. Populating the cache first ensures that this work only
    # happens once.
    needs: generate_cache

    strategy:
      # By default, this is set to `true`, which means that a single CI job
      # failure will cause all outstanding jobs to be canceled. This slows down
      # development because it means that errors need to be encountered and
      # fixed one at a time.
      fail-fast: false
      matrix:
        # See `INTERNAL.md` for an explanation of these pinned toolchain
        # versions.
        toolchain: [ "msrv", "stable", "nightly",  "zerocopy-generic-bounds-in-const-fn", "zerocopy-aarch64-simd", "zerocopy-panic-in-const", ]
        target: [
          "i686-unknown-linux-gnu",
          "x86_64-unknown-linux-gnu",
          "arm-unknown-linux-gnueabi",
          "aarch64-unknown-linux-gnu",
          "powerpc-unknown-linux-gnu",
          "powerpc64-unknown-linux-gnu",
          "riscv64gc-unknown-linux-gnu",
          "s390x-unknown-linux-gnu",
          "wasm32-wasi"
        ]
        features: [ "--no-default-features", "", "--features __internal_use_only_features_that_work_on_stable", "--all-features" ]
        crate: [ "zerocopy", "zerocopy-derive" ]
        exclude:
          # Exclude any combination which uses a non-nightly toolchain but
          # enables nightly features.
          - toolchain: "msrv"
            features: "--all-features"
          - toolchain: "stable"
            features: "--all-features"
          - toolchain: "zerocopy-generic-bounds-in-const-fn"
            features: "--all-features"
          - toolchain: "zerocopy-aarch64-simd"
            features: "--all-features"
          - toolchain: "zerocopy-panic-in-const"
            features: "--all-features"

          # Exclude any combination for the zerocopy-derive crate which
          # uses zerocopy features.
          - crate: "zerocopy-derive"
            features: "--no-default-features"
          - crate: "zerocopy-derive"
            features: "--features __internal_use_only_features_that_work_on_stable"
          - crate: "zerocopy-derive"
            features: "--all-features"
          # Exclue any combination of zerocopy-derive and any toolchain version
          # other than "msrv", "stable", and "nightly". These other versions
          # exist to exercise zerocopy behavior which differs by toolchain;
          # zerocopy-derive doesn't behave different on these toolchains.
          - crate: "zerocopy-derive"
            toolchain: "zerocopy-generic-bounds-in-const-fn"
          - crate: "zerocopy-derive"
            toolchain: "zerocopy-aarch64-simd"
          - crate: "zerocopy-derive"
            toolchain: "zerocopy-panic-in-const"

    name: Build & Test (${{ matrix.crate }} / ${{ matrix.toolchain }} / ${{ matrix.features }} / ${{ matrix.target }})

    steps:
    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

    - name: Populate cache
      uses: ./.github/actions/cache
    
    - name: Configure environment variables
      run: |
        set -eo pipefail

        # We use toolchain descriptors ("msrv", "stable", "nightly", and values
        # from the "metadata.build-rs" key in Cargo.toml) in the matrix. This
        # step converts the current descriptor to a particular toolchain version
        # by looking up the corresponding key in `Cargo.toml`. It sets the
        # `ZC_TOOLCHAIN` environment variable for use in the next step
        # (toolchain installation) because GitHub variable interpolation doesn't
        # support running arbitrary commands. In other words, we can't rewrite:
        #
        #   toolchain: $ {{ env.ZC_TOOLCHAIN }}
        #
        # ...to:
        #
        #   toolchain: $ {{ ./cargo.sh --version matrix.toolchain }} # hypothetical syntax
        ZC_TOOLCHAIN="$(./cargo.sh --version ${{ matrix.toolchain }})"
        echo "Found that the '${{ matrix.toolchain }}' toolchain is $ZC_TOOLCHAIN" | tee -a $GITHUB_STEP_SUMMARY
        echo "ZC_TOOLCHAIN=$ZC_TOOLCHAIN" >> $GITHUB_ENV

        if [[ '${{ matrix.toolchain }}' == 'nightly' ]]; then
          RUSTFLAGS="$RUSTFLAGS $ZC_NIGHTLY_RUSTFLAGS"
          MIRIFLAGS="$MIRIFLAGS $ZC_NIGHTLY_MIRIFLAGS"
          echo "Using nightly toolchain; setting RUSTFLAGS='$RUSTFLAGS' and MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY
          echo "RUSTFLAGS=$RUSTFLAGS" >> $GITHUB_ENV
          echo "MIRIFLAGS=$MIRIFLAGS" >> $GITHUB_ENV
        else
          echo "Using non-nightly toolchain; not modifying RUSTFLAGS='$RUSTFLAGS' or MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY
        fi

    # On our MSRV, `cargo` does not know about the `rust-version` field. As a
    # result, in `cargo.sh`, if we use our MSRV toolchain in order to run `cargo
    # metadata`, we will not be able to extract the `rust-version` field. Thus,
    # in `cargo.sh`, we explicitly do `cargo +stable metadata`. This requires a
    # (more recent) stable toolchain to be installed. As of this writing, this
    # toolchain is not used for anything else.
    - name: Install stable Rust for use in 'cargo.sh'
      uses: dtolnay/rust-toolchain@00b49be78f40fba4e87296b2ead62868750bdd83 # stable
      with:
        toolchain: stable

    - name: Install Rust with ${{ matrix.toolchain }} toolchain (${{ env.ZC_TOOLCHAIN }}) and target ${{ matrix.target }}
      uses: dtolnay/rust-toolchain@00b49be78f40fba4e87296b2ead62868750bdd83 # stable
      with:
          toolchain: ${{ env.ZC_TOOLCHAIN }}
          targets: ${{ matrix.target }}
          # We require the `rust-src` component to ensure that the compiler
          # error output generated during UI tests matches that generated on
          # local developer machines; see
          # https://github.com/rust-lang/rust/issues/116433.
          #
          # Only nightly has a working Miri, so we skip installing on all other
          # toolchains. This expression is effectively a ternary expression -
          # see [1] for details.
          #
          # [1] https://github.com/actions/runner/issues/409#issuecomment-752775072
          components: clippy, rust-src ${{ matrix.toolchain == 'nightly' && ', miri' || '' }}

    - name: Rust Cache
      uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
      with:
        key: "${{ matrix.target }}"

    - name: Check tests
      run: ./cargo.sh +${{ matrix.toolchain }} check --tests --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose

    - name: Build
      run: ./cargo.sh +${{ matrix.toolchain }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose

    # When building tests for the i686 target, we need certain libraries which
    # are not installed by default; `gcc-multilib` includes these libraries.
    - name: Install gcc-multilib
      # Per [1]:
      #
      #   Note: Always run `sudo apt-get update` before installing a package. In
      #   case the `apt` index is stale, this command fetches and re-indexes any
      #   available packages, which helps prevent package installation failures.
      #
      # [1] https://docs.github.com/en/actions/using-github-hosted-runners/customizing-github-hosted-runners
      run: |
        sudo apt-get update
        sudo apt-get install gcc-multilib
      if: contains(matrix.target, 'i686')

    - name: Run tests
      run: |
        ./cargo.sh +${{ matrix.toolchain }} test \
          --package ${{ matrix.crate }} \
          --target ${{ matrix.target }} \
          ${{ matrix.features }} \
          --verbose \
          -- \
          --skip ui

      # Only run tests when targetting x86 (32- or 64-bit) - we're executing on
      # x86_64, so we can't run tests for any non-x86 target.
      #
      # TODO(https://github.com/dtolnay/trybuild/issues/184#issuecomment-1269097742):
      # Run compile tests when building for other targets.
      if: contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')

    - name: Run UI tests
      run: |
        # Run UI tests separately, treating warnings as warnings (rather than
        # as errors, as we do everywhere else in our CI tests). This allows
        # our UI tests to more accurately reflect what users will see, and
        # also ensures that we're not spuriously relying on warnings being
        # errors to ensure compilation failure (if we were, then our code
        # would be unsound whenever -Dwarnings is not enabled).
        #
        # TODO(#560), TODO(#187): Once we migrate to the ui-test crate, we
        # likely won't have to special-case the UI tests like this.
        RUSTFLAGS="$RUSTFLAGS -Wwarnings" ./cargo.sh +${{ matrix.toolchain }} test \
          --package ${{ matrix.crate }} \
          --target ${{ matrix.target }} \
          ${{ matrix.features }} \
          --verbose \
          ui

      # Only run tests when targetting x86 (32- or 64-bit) - we're executing on
      # x86_64, so we can't run tests for any non-x86 target.
      #
      # TODO(https://github.com/dtolnay/trybuild/issues/184#issuecomment-1269097742):
      # Run compile tests when building for other targets.
      #
      # Only run UI tests for zerocopy-derive, or for zerocopy with the derive
      # feature.
      #
      # Only run UI tests for the 'msrv', 'stable', and 'nightly' toolchains.
      # Other toolchains are tested only because zerocopy has behavior which
      # differs on those toolchains, but at present, none of that behavior
      # affects UI tests. If we were to run UI tests on these toolchains, we
      # would need a new set of UI test files per toolchain, which would be a
      # maintenance burden not worth the (at present, zero) benefit.
      if: |
        (contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')) &&
          (matrix.crate == 'zerocopy-derive' ||
            (matrix.features != '' && matrix.features != '--no-default-features')) &&
          (matrix.toolchain == 'msrv' || matrix.toolchain == 'stable' || matrix.toolchain == 'nightly')

    - name: Run tests under Miri
      run: |
        set -eo pipefail

        # Work around https://github.com/rust-lang/miri/issues/3125
        [ "${{ matrix.target }}" == "aarch64-unknown-linux-gnu" ] && cargo clean

        # We run two executions in parallel (one for each aliasing model), so
        # this has the effect of, in total, running twice the number of workers
        # as there are CPU cores. Note that this isn't *quite* the same as
        # having 2n threads, since the two distinct executions can't share work.
        # Hopefully, since the executions should be nearly identical (I don't
        # assume that the aliasing model has a huge impact on the performance
        # characteristics of these tests), this shouldn't be a big deal.
        THREADS=$(nproc)
        echo "Running Miri tests with $THREADS threads per aliasing model" | tee -a $GITHUB_STEP_SUMMARY

        cargo install cargo-nextest

        # Run under both the stacked borrows model (default) and under the tree 
        # borrows model to ensure we're compliant with both.
        #
        # Run both executions in parallel to make better use of available
        # parallelism.
        for EXTRA_FLAGS in "" "-Zmiri-tree-borrows"; do
          MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" ./cargo.sh +${{ matrix.toolchain }} \
            miri nextest run \
            --test-threads "$THREADS" \
            --package ${{ matrix.crate }} \
            --target ${{ matrix.target }} \
            ${{ matrix.features }} &
        done

        wait

      # Only nightly has a working Miri, so we skip installing on all other
      # toolchains.
      #
      # TODO(#22): Re-enable testing on riscv64gc-unknown-linux-gnu and/or
      # wasm32-wasi once those work.
      if: matrix.toolchain == 'nightly' && matrix.target != 'riscv64gc-unknown-linux-gnu' && matrix.target != 'wasm32-wasi'

    - name: Clippy check
      run: ./cargo.sh +${{ matrix.toolchain }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose
      # Clippy improves the accuracy of lints over time, and fixes bugs. Only
      # running Clippy on nightly allows us to avoid having to write code which
      # is compatible with older versions of Clippy, which sometimes requires
      # hacks to work around limitations that are fixed in more recent versions.
      if: matrix.toolchain == 'nightly'

    - name: Cargo doc
      # We pass --document-private-items and --document-hidden items to ensure that 
      # documentation always builds even for these items. This makes future changes to 
      # make those items public/non-hidden more painless. Note that 
      # --document-hidden-items is unstable; if a future release breaks or removes it,
      # we can just update CI to no longer pass that flag.
      run: |
        # Include arguments passed during docs.rs deployments to make sure those
        # work properly.
        METADATA_DOCS_RS_RUSTDOC_ARGS="$(cargo metadata --format-version 1 | \
          jq -r ".packages[] | select(.name == \"zerocopy\").metadata.docs.rs.\"rustdoc-args\".[]" | tr '\n' ' ')"
        export RUSTDOCFLAGS="${{ matrix.toolchain == 'nightly' && '-Z unstable-options --document-hidden-items' || '' }} $RUSTDOCFLAGS $METADATA_DOCS_RS_RUSTDOC_ARGS"
        ./cargo.sh +${{ matrix.toolchain }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }}

    # Check semver compatibility with the most recently-published version on
    # crates.io. We do this in the matrix rather than in its own job so that it
    # gets run on different targets. Some of our API is target-specific (e.g.,
    # SIMD type impls), and so we need to run on each target.
    #
    # TODO(https://github.com/obi1kenobi/cargo-semver-checks-action/issues/54):
    # Currently we don't actually do anything with `matrix.target`, so we're
    # just duplicating work by running this job multiple times, each time
    # targetting the host platform.
    - name: Check semver compatibility
      uses: obi1kenobi/cargo-semver-checks-action@c7306483f698c511eaf7416d1bf2e1958c90140f # v2.4
      with:
        # Don't semver check zerocopy-derive; as a proc macro, it doesn't have
        # an API that cargo-semver-checks can understand.
        package: zerocopy
        # Don't test nightly features. The reason this doesn't work is that
        # cargo-semver-checks works relative to the most recently-published
        # stable release on crates.io. The pinned nightly toolchain may be
        # incompatible with the unstable features on that published release, as
        # nightly features may have changed since it was released. Luckily, we
        # don't care about the API stability of these features anyway.
        feature-group: only-explicit-features
        features: __internal_use_only_features_that_work_on_stable
        rust-toolchain: ${{ env.ZC_TOOLCHAIN }}
      if: matrix.crate == 'zerocopy' && matrix.features == '--features __internal_use_only_features_that_work_on_stable' && matrix.toolchain == 'nightly'

    # TODO(#453): Doing this as a matrix step is a hack that allows us to depend
    # on the fact that toolchains have already been installed. We currently only
    # run this on a single matrix combination, but doing it outside of the
    # matrix would require us to replicate the toolchain resolution and
    # installation logic. We should either:
    # - Figure out how to factor out the toolchain resolution and installation
    #   logic (see #1275) for an attempt
    # - Support multiple matrix combinations (which we intend to do as part of
    #   #453 eventually anyway) so that this location is justified
    - name: Generate code coverage
      run: |
        set -eo pipefail
        ./cargo.sh +nightly install cargo-llvm-cov

        ./cargo.sh +${{ matrix.toolchain }} llvm-cov \
          --package ${{ matrix.crate }}              \
          --target ${{ matrix.target }}              \
          ${{ matrix.features }}                     \
          --lcov                                     \
          --output-path lcov.info                    \
          --verbose                                  \
          --                                         \
          --skip ui
      if: |
        matrix.crate == 'zerocopy'          &&
        matrix.features == '--all-features' &&
        matrix.toolchain == 'nightly'       &&
        matrix.target == 'x86_64-unknown-linux-gnu'

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        files: lcov.info
      if: |
        matrix.crate == 'zerocopy'          &&
        matrix.features == '--all-features' &&
        matrix.toolchain == 'nightly'       &&
        matrix.target == 'x86_64-unknown-linux-gnu'

  kani:
    runs-on: ubuntu-latest        
    name: 'Run tests under Kani'
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: model-checking/kani-github-action@f838096619a707b0f6b2118cf435eaccfa33e51f # v1.1
        with:
          # Use `--features __internal_use_only_features_that_work_on_stable`
          # because the Kani GitHub Action uses its own pinned nightly
          # toolchain. Sometimes, we make changes to our nightly features for
          # more recent toolchains, and so our nightly features become
          # incompatible with the toolchain that Kani uses. By only testing
          # stable features, we ensure that this doesn't cause problems in CI.
          #
          # TODO(https://github.com/model-checking/kani-github-action/issues/56):
          # Go back to testing all features once the Kani GitHub Action supports
          # specifying a particular toolchain.
          args: "--package zerocopy --features __internal_use_only_features_that_work_on_stable --output-format=terse --randomize-layout --memory-safety-checks --overflow-checks --undefined-function-checks --unwinding-checks"
          # This version is automatically rolled by
          # `roll-pinned-toolchain-versions.yml`.
          kani-version: 0.51.0

  check_fmt:
    runs-on: ubuntu-latest
    name: Check Rust formatting
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Check Rust formatting
        run: ./ci/check_fmt.sh

  check_readme:
    needs: generate_cache
    runs-on: ubuntu-latest
    name: Check README.md
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

      - name: Populate cache
        uses: ./.github/actions/cache
      
      - name: Check README.md
        run: ./ci/check_readme.sh

  check_msrv:
    needs: generate_cache
    runs-on: ubuntu-latest
    name: Check MSRVs match
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      
      - name: Populate cache
        uses: ./.github/actions/cache
      
      # Make sure that the MSRV in zerocopy's and zerocopy-derive's `Cargo.toml`
      # files are the same. In CI, we test with a single MSRV (the one indicated
      # in zerocopy's `Cargo.toml`), so it's important that:
      # - zerocopy-derive's MSRV is not lower than zerocopy's (we don't test with
      #   a lower MSRV in CI, so we couldn't guarantee that zerocopy-derive
      #   actually built and ran on a lower MSRV)
      # - zerocopy-derive's MSRV is not higher than zerocopy's (this would mean
      #   that compiling zerocopy with the `derive` feature enabled would fail
      #   on its own published MSRV)
      - name: Check MSRVs match
        run: ./ci/check_msrv.sh

  check_versions:
    needs: generate_cache
    runs-on: ubuntu-latest
    name: Check crate versions match
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

      - name: Populate cache
        uses: ./.github/actions/cache

      # Make sure that both crates are at the same version, and that zerocopy
      # depends exactly upon the current version of zerocopy-derive. See
      # `INTERNAL.md` for an explanation of why we do this.
      - name: Check crate versions match
        run: ./ci/check_versions.sh

  generate_cache:
    runs-on: ubuntu-latest
    name: Generate cache
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

      - name: Populate cache
        uses: ./.github/actions/cache

      - name: Download dependencies
        run: |
          # Ensure all dependencies are downloaded - both for our crates and for
          # tools we use in CI. We don't care about these tools succeeding for
          # two reasons: First, this entire job is best-effort since it's just a
          # performance optimization. Second, there may be failures due to
          # issues other than failing to download dependencies (e.g., `cargo
          # metadata` called with a malformed `Cargo.toml`, build failure in our
          # own crate or in dependencies, etc). For those reasons, we discard
          # stderr and ignore status codes.
          #
          # For downloading our crates' dependencies in particular, note that
          # there is no support for doing this directly [1], so we just check
          # all crates using --tests.
          #
          # We background all jobs and then wait for them so that they can run
          # in parallel.
          #
          # [1]  https://stackoverflow.com/a/42139535/836390
          cargo check --workspace --tests            &> /dev/null &
          cargo metadata                             &> /dev/null &
          cargo install cargo-readme --version 3.2.0 &> /dev/null &
          cargo install --locked kani-verifier       &> /dev/null &
          cargo install cargo-nextest                &> /dev/null &
          cargo kani setup                           &> /dev/null &

          wait

  check-all-toolchains-tested:
    runs-on: ubuntu-latest
    name: Check that all toolchains listed in Cargo.toml are tested in CI
    steps:
      - name: Install yq (for YAML parsing)
        run: go install github.com/mikefarah/yq/v4@latest
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Run check
        run: ./ci/check_all_toolchains_tested.sh

  check-job-dependencies:
    runs-on: ubuntu-latest
    name: Check all-jobs-succeeded depends on all jobs
    steps:
      - name: Install yq (for YAML parsing)
        run: go install github.com/mikefarah/yq/v4@latest
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Run dependency check
        run: ./ci/check_job_dependencies.sh

  run-git-hooks:
    runs-on: ubuntu-latest
    name: Run Git hooks
    steps:
      - name: Install yq (for YAML parsing)
        run: go install github.com/mikefarah/yq/v4@latest
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Run dependency check
        # Ensure that Git hooks execute successfully.
        #
        # Note that this runs all hooks. As of this writing, the only Git hook
        # is `pre-push`. Running all hooks ensures that, if a new hook is added
        # which can't run in CI (at least not using this naive setup), we'll
        # notice and it will remind us to update this test.
        run: set -e; for hook in ./githooks/*; do $hook; done

  # Used to signal to branch protections that all other jobs have succeeded.
  all-jobs-succeed:
      name: All checks succeeded
      # On failure, we run and unconditionally exit with a failing status code.
      # On success, this job is skipped. Jobs skipped using `if:` are considered
      # to have succeeded:
      #
      # https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks
      if: failure()
      runs-on: ubuntu-latest
      needs: [build_test, kani, check_fmt, check_readme, check_msrv, check_versions, generate_cache, check-all-toolchains-tested, check-job-dependencies, run-git-hooks]
      steps:
        - name: Mark the job as failed
          run: exit 1