diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index cfba6d7225b8..0070899e9774 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -21,10 +21,12 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} permissions: read-all env: + TRITON_BUILD_WITH_CCACHE: "true" TRITON_BUILD_WITH_CLANG_LLD: "TRUE" TRITON_USE_ASSERT_ENABLED_LLVM: "TRUE" TRITON_DISABLE_LINE_INFO: 1 PROTON_SKIP_PC_SAMPLING_TEST: 1 + CCACHE_COMPRESS: "true" jobs: Runner-Preparation: runs-on: ubuntu-latest @@ -154,6 +156,8 @@ jobs: strategy: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-CUDA)}} + env: + RUNNER_TYPE: ${{ matrix.runner[0] }} steps: - name: Checkout uses: actions/checkout@v4 @@ -199,22 +203,30 @@ jobs: # "restore" step. This is to prevent the caches from accumulating stale # files over time. name: Restore cache of ccache and Triton compilation artifacts - if: github.event_name != 'push' + id: restore-build-cache + if: github.ref != 'refs/heads/main' uses: actions/cache/restore@v4 with: path: | ~/.triton/cache - ~/.cache/ccache + ~/.ccache # Restore the most recent cache entry. - restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}- + restore-keys: | + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}- + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}- # We expect this cache key never to hit and for us to fall back # unconditionally to the restore-key, so it doesn't actually matter # what we put here (so long as it doesn't hit an existing key). - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directory + key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} + - name: Inspect cache directories run: | mkdir -p ~/.triton ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - name: Update PATH run: | echo "$HOME/.local/bin" >> $GITHUB_PATH @@ -224,12 +236,14 @@ jobs: python3 -m pip install cython setuptools wheel cmake==3.24 ninja pytest-forked pytest-xdist lit - name: Install Triton env: - TRITON_BUILD_WITH_CCACHE: "true" CUDA_HOME: "/usr/local/cuda" run: | echo "PATH is '$PATH'" cd python - python3 -m pip install '.[tests]' + ccache --zero-stats + python3 -m pip install -v '.[tests]' + - name: CCache Stats + run: ccache --print-stats - name: Run lit tests run: | cd python @@ -278,6 +292,15 @@ jobs: cd third_party/proton/test python3 -m pytest -s . cd .. + - name: Inspect cache directories + run: | + mkdir -p ~/.triton + ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - # If we're on branch `main`, save the ccache Triton compilation artifacts # to the cache so they can be used by other (non-main) CI runs. # @@ -287,22 +310,17 @@ jobs: if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: - path: ~/.triton/cache ~/.cache/ccache - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directories - run: | - mkdir -p ~/.triton - ls -alh ~/.triton - du -sh ~/.triton/** - - mkdir -p ~/.cache/ccache - ls -alh ~/.cache/ccache - du -sh ~/.cache/ccache + path: | + ~/.triton/cache + ~/.ccache + key: ${{ steps.restore-build-cache.outputs.cache-primary-key }} Integration-Tests-AMD: needs: Runner-Preparation if: needs.Runner-Preparation.outputs.matrix-HIP != '' runs-on: ${{ matrix.runner }} timeout-minutes: 30 + env: + RUNNER_TYPE: ${{ matrix.runner[1] }} strategy: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-HIP)}} @@ -355,22 +373,30 @@ jobs: # "restore" step. This is to prevent the caches from accumulating stale # files over time. name: Restore cache of ccache and Triton compilation artifacts - if: github.event_name != 'push' + id: restore-build-cache + if: github.ref != 'refs/heads/main' uses: actions/cache/restore@v4 with: path: | ~/.triton/cache - ~/.cache/ccache + ~/.ccache # Restore the most recent cache entry. - restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}- + restore-keys: | + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}- + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}- # We expect this cache key never to hit and for us to fall back # unconditionally to the restore-key, so it doesn't actually matter # what we put here (so long as it doesn't hit an existing key). - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directory + key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} + - name: Inspect cache directories run: | mkdir -p ~/.triton ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - name: Update PATH run: | echo "/opt/rocm/llvm/bin" >> $GITHUB_PATH @@ -378,17 +404,24 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install lit + - name: Install apt dependencies + run: | + apt update + apt install ccache - name: Install Triton id: amd-install-triton run: | echo "PATH is '$PATH'" pip uninstall -y triton cd python + ccache --zero-stats pip install -v -e '.[tests]' - name: Clean up after an unsuccessful build if: ${{ !success() && steps.amd-install-triton.outcome != 'success' }} run: | rm -rf ~/.triton + - name: CCache Stats + run: ccache --print-stats - name: Run lit tests run: | cd python @@ -431,6 +464,15 @@ jobs: cd python cd "build/$(ls build | grep -i cmake)" ctest -j32 + - name: Inspect cache directories + run: | + mkdir -p ~/.triton + ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - # If we're on branch `main`, save the ccache Triton compilation artifacts # to the cache so they can be used by other (non-main) CI runs. # @@ -440,17 +482,10 @@ jobs: if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: - path: ~/.triton/cache ~/.cache/ccache - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directories - run: | - mkdir -p ~/.triton - ls -alh ~/.triton - du -sh ~/.triton/** - - mkdir -p ~/.cache/ccache - ls -alh ~/.cache/ccache - du -sh ~/.cache/ccache + path: | + ~/.triton/cache + ~/.ccache + key: ${{ steps.restore-build-cache.outputs.cache-primary-key }} - name: Clean up caches run: | rm -rf ~/.triton/cache @@ -462,6 +497,8 @@ jobs: strategy: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-MACOS)}} + env: + RUNNER_TYPE: ${{ matrix.runner[0] }} steps: - name: Checkout uses: actions/checkout@v4 @@ -470,7 +507,7 @@ jobs: - name: Install brew dependencies run: | brew update - brew install ccache llvm@19 lld + brew install ccache llvm@19 lld coreutils - name: Compute cache keys id: cache-key run: | @@ -511,22 +548,30 @@ jobs: # "restore" step. This is to prevent the caches from accumulating stale # files over time. name: Restore cache of ccache and Triton compilation artifacts - if: github.event_name != 'push' + id: restore-build-cache + if: github.ref != 'refs/heads/main' uses: actions/cache/restore@v4 with: path: | ~/.triton/cache - ~/.cache/ccache + ~/.ccache # Restore the most recent cache entry. - restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}- + restore-keys: | + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}- + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}- # We expect this cache key never to hit and for us to fall back # unconditionally to the restore-key, so it doesn't actually matter # what we put here (so long as it doesn't hit an existing key). - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directory + key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} + - name: Inspect cache directories run: | mkdir -p ~/.triton ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - name: Update PATH run: | echo "$HOME/.local/bin" >> $GITHUB_PATH @@ -539,7 +584,6 @@ jobs: python3 -m pip install cython setuptools wheel cmake==3.24 ninja pytest-xdist lit pybind11 - name: Install Triton env: - TRITON_BUILD_WITH_CCACHE: "true" TRITON_BUILD_WITH_O1: "true" # macos-latest has 3 vcpus and 7GB DRAM, to save memory we limit the number of jobs to 3 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories @@ -548,7 +592,19 @@ jobs: source ~/.venv/bin/activate echo "PATH is '$PATH'" cd python - python3 -m pip install --no-build-isolation . + ccache --zero-stats + python3 -m pip install -v --no-build-isolation . + - name: CCache Stats + run: ccache --print-stats + - name: Inspect cache directories + run: | + mkdir -p ~/.triton + ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - # If we're on branch `main`, save the ccache Triton compilation artifacts # to the cache so they can be used by other (non-main) CI runs. # @@ -558,14 +614,7 @@ jobs: if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: - path: ~/.triton/cache ~/.cache/ccache - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - name: Inspect cache directories - run: | - mkdir -p ~/.triton - ls -alh ~/.triton - du -sh ~/.triton/** - - mkdir -p ~/.cache/ccache - ls -alh ~/.cache/ccache - du -sh ~/.cache/ccache + path: | + ~/.triton/cache + ~/.ccache + key: ${{ steps.restore-build-cache.outputs.cache-primary-key }} diff --git a/.github/workflows/integration-tests.yml.in b/.github/workflows/integration-tests.yml.in index 7da4aa079327..4404b2aa6052 100644 --- a/.github/workflows/integration-tests.yml.in +++ b/.github/workflows/integration-tests.yml.in @@ -23,10 +23,12 @@ concurrency: permissions: read-all env: + TRITON_BUILD_WITH_CCACHE: "true" TRITON_BUILD_WITH_CLANG_LLD: "TRUE" TRITON_USE_ASSERT_ENABLED_LLVM: "TRUE" TRITON_DISABLE_LINE_INFO: 1 PROTON_SKIP_PC_SAMPLING_TEST: 1 + CCACHE_COMPRESS: "true" jobs: Runner-Preparation: @@ -174,6 +176,9 @@ jobs: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-CUDA)}} + env: + RUNNER_TYPE: ${{ matrix.runner[0] }} + steps: - name: Checkout uses: actions/checkout@v4 @@ -225,24 +230,32 @@ jobs: # files over time. - &restore-build-artifacts-step name: Restore cache of ccache and Triton compilation artifacts - if: github.event_name != 'push' + id: restore-build-cache + if: github.ref != 'refs/heads/main' uses: actions/cache/restore@v4 with: path: | ~/.triton/cache - ~/.cache/ccache + ~/.ccache # Restore the most recent cache entry. - restore-keys: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}- + restore-keys: | + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}- + triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}- # We expect this cache key never to hit and for us to fall back # unconditionally to the restore-key, so it doesn't actually matter # what we put here (so long as it doesn't hit an existing key). - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} + key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ env.RUNNER_TYPE }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - &inspect-cache-directory-step - name: Inspect cache directory + - &inspect-cache-directories-step + name: Inspect cache directories run: | mkdir -p ~/.triton ls -alh ~/.triton + du -sh ~/.triton/** + + mkdir -p ~/.ccache + ls -alh ~/.ccache + du -sh ~/.ccache - name: Update PATH run: | @@ -255,12 +268,16 @@ jobs: - name: Install Triton env: - TRITON_BUILD_WITH_CCACHE: "true" CUDA_HOME: "/usr/local/cuda" run: | echo "PATH is '$PATH'" cd python - python3 -m pip install '.[tests]' + ccache --zero-stats + python3 -m pip install -v '.[tests]' + + - &print-ccache-stats + name: CCache Stats + run: ccache --print-stats - &run-lit-tests-step name: Run lit tests @@ -319,6 +336,8 @@ jobs: python3 -m pytest -s . cd .. + - *inspect-cache-directories-step + # If we're on branch `main`, save the ccache Triton compilation artifacts # to the cache so they can be used by other (non-main) CI runs. # @@ -329,19 +348,10 @@ jobs: if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: - path: ~/.triton/cache ~/.cache/ccache - key: triton-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ runner.name }}-llvm-${{ steps.cache-key.outputs.llvm }}-${{ steps.cache-key.outputs.datetime }} - - - &inspect-cache-directories-step - name: Inspect cache directories - run: | - mkdir -p ~/.triton - ls -alh ~/.triton - du -sh ~/.triton/** - - mkdir -p ~/.cache/ccache - ls -alh ~/.cache/ccache - du -sh ~/.cache/ccache + path: | + ~/.triton/cache + ~/.ccache + key: ${{ steps.restore-build-cache.outputs.cache-primary-key }} Integration-Tests-AMD: needs: Runner-Preparation @@ -350,6 +360,9 @@ jobs: runs-on: ${{ matrix.runner }} timeout-minutes: 30 + env: + RUNNER_TYPE: ${{ matrix.runner[1] }} + strategy: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-HIP)}} @@ -369,7 +382,7 @@ jobs: - *compute-cache-keys-step - *cache-build-dependencies-step - *restore-build-artifacts-step - - *inspect-cache-directory-step + - *inspect-cache-directories-step - name: Update PATH run: | @@ -380,12 +393,18 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install lit + - name: Install apt dependencies + run: | + apt update + apt install ccache + - name: Install Triton id: amd-install-triton run: | echo "PATH is '$PATH'" pip uninstall -y triton cd python + ccache --zero-stats pip install -v -e '.[tests]' - name: Clean up after an unsuccessful build @@ -393,6 +412,7 @@ jobs: run: | rm -rf ~/.triton + - *print-ccache-stats - *run-lit-tests-step - name: Run python tests on HIP @@ -423,8 +443,8 @@ jobs: - *run-proton-tests-step - *run-cpp-unittests-step - - *save-build-artifacts-step - *inspect-cache-directories-step + - *save-build-artifacts-step - name: Clean up caches run: | @@ -438,6 +458,10 @@ jobs: strategy: matrix: runner: ${{fromJson(needs.Runner-Preparation.outputs.matrix-MACOS)}} + + env: + RUNNER_TYPE: ${{ matrix.runner[0] }} + steps: - name: Checkout uses: actions/checkout@v4 @@ -446,12 +470,12 @@ jobs: - name: Install brew dependencies run: | brew update - brew install ccache llvm@19 lld + brew install ccache llvm@19 lld coreutils - *compute-cache-keys-step - *cache-build-dependencies-step - *restore-build-artifacts-step - - *inspect-cache-directory-step + - *inspect-cache-directories-step - name: Update PATH run: | @@ -465,7 +489,6 @@ jobs: python3 -m pip install cython setuptools wheel cmake==3.24 ninja pytest-xdist lit pybind11 - name: Install Triton env: - TRITON_BUILD_WITH_CCACHE: "true" TRITON_BUILD_WITH_O1: "true" # macos-latest has 3 vcpus and 7GB DRAM, to save memory we limit the number of jobs to 3 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories @@ -474,7 +497,9 @@ jobs: source ~/.venv/bin/activate echo "PATH is '$PATH'" cd python - python3 -m pip install --no-build-isolation . + ccache --zero-stats + python3 -m pip install -v --no-build-isolation . - - *save-build-artifacts-step + - *print-ccache-stats - *inspect-cache-directories-step + - *save-build-artifacts-step