Skip to content

Commit

Permalink
[ci] Add lint job and enable yaml linting of GH files. (#2819)
Browse files Browse the repository at this point in the history
  • Loading branch information
@stellaraccident
stellaraccident committed Jan 27, 2024
1 parent 28c7051 commit 4a4d80a
Show file tree
Hide file tree
Showing 10 changed files with 642 additions and 620 deletions.
99 changes: 50 additions & 49 deletions .github/actions/setup-build/action.yml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# yamllint disable rule:line-length
name: "Setup build environment"
description: "Setup the build environment. An action so that it can be shared between in-tree/out-of-tree jobs"

Expand All @@ -24,59 +25,59 @@ runs:
using: "composite"

steps:
- name: Set up Python
if: ${{ runner.arch == 'X64' }}
uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Set up Python
if: ${{ runner.arch == 'X64' }}
uses: actions/setup-python@v4
with:
python-version: '3.11'

- name: Install MLIR Python depends
if: ${{ runner.os != 'Linux' }}
run: |
python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt
shell: bash
- name: Install MLIR Python depends
if: ${{ runner.os != 'Linux' }}
run: |
python -m pip install -r $GITHUB_WORKSPACE/externals/llvm-project/mlir/python/requirements.txt
shell: bash

- name: Install PyTorch nightly depends
if: ${{ runner.os != 'Linux' }}
run: |
python -m pip install -r pytorch-requirements.txt
python -m pip install -r build-requirements.txt
shell: bash
- name: Install PyTorch nightly depends
if: ${{ runner.os != 'Linux' }}
run: |
python -m pip install -r pytorch-requirements.txt
python -m pip install -r build-requirements.txt
shell: bash

- name: Install prerequisites (Linux)
if: ${{ runner.os == 'Linux' }}
run: sudo apt-get install --yes ccache ninja-build
shell: bash
- name: Install prerequisites (Linux)
if: ${{ runner.os == 'Linux' }}
run: sudo apt-get install --yes ccache ninja-build
shell: bash

- name: Install prerequisites (macOS)
if: ${{ runner.os == 'macOS' }}
run: brew install ccache ninja
shell: bash
- name: Install prerequisites (macOS)
if: ${{ runner.os == 'macOS' }}
run: brew install ccache ninja
shell: bash

- name: Install prerequisites (Windows)
if: ${{ runner.os == 'Windows' }}
run: |
pip install ninja
choco install ccache --yes
shell: bash
- name: Install prerequisites (Windows)
if: ${{ runner.os == 'Windows' }}
run: |
pip install ninja
choco install ccache --yes
shell: bash

- name: Configure ccache
if: ${{ inputs.cache-enabled == 'true' }}
run: |
rm -rf ${{ github.workspace }}/.ccache
mkdir -p ${{ github.workspace }}/.ccache
ccache --set-config "cache_dir=${{ github.workspace }}/.ccache"
ccache --set-config "compression=true"
ccache --set-config "max_size=300M"
ccache --zero-stats
shell: bash
- name: Configure ccache
if: ${{ inputs.cache-enabled == 'true' }}
run: |
rm -rf ${{ github.workspace }}/.ccache
mkdir -p ${{ github.workspace }}/.ccache
ccache --set-config "cache_dir=${{ github.workspace }}/.ccache"
ccache --set-config "compression=true"
ccache --set-config "max_size=300M"
ccache --zero-stats
shell: bash

- name: Enable ccache
if: ${{ inputs.cache-enabled == 'true' }}
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/.ccache
key: ${{ runner.os }}-${{ inputs.cache-suffix }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ inputs.cache-suffix }}-
${{ runner.os }}-
- name: Enable ccache
if: ${{ inputs.cache-enabled == 'true' }}
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/.ccache
key: ${{ runner.os }}-${{ inputs.cache-suffix }}-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ inputs.cache-suffix }}-
${{ runner.os }}-
260 changes: 130 additions & 130 deletions .github/workflows/RollPyTorch.yml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
# yamllint disable rule:line-length
name: Roll PyTorch

on:
Expand All @@ -14,133 +15,132 @@ jobs:
if: github.repository == 'llvm/torch-mlir'

steps:

- name: Prepare workspace
run: |
# Clear the workspace directory so that we don't run into errors about
# existing lock files.
sudo rm -rf $GITHUB_WORKSPACE/*
- name: Get torch-mlir
uses: actions/checkout@v3
with:
submodules: 'false'
token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}

- name: Get LLVM and StableHlo submodules
run: |
set -eo pipefail
cd ${GITHUB_WORKSPACE}
# Fetching the submodules concurrently may cause problems, so we fetch
# them one after another.
rm -f .git/modules/externals/llvm-project/index.lock
rm -f .git/modules/externals/stablehlo/index.lock
git submodule update --init --recursive externals/llvm-project
git submodule update --init --recursive externals/stablehlo
- name: Setup ccache
uses: ./.github/actions/setup-build
with:
cache-suffix: 'rollPyTorch'

- name: Determine nightly PyTorch version
run: |
set -eo pipefail
cd ${GITHUB_WORKSPACE}
python -m pip install wheel
sudo apt-get install unzip
# Fetch the most recent nightly torchvision release
VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
echo "Found torchvision release ${VISION_RELEASE}"
# Fetch the whl file associated with the nightly torchvision release
rm -f torch*.whl
python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}"
# Downloading the torchvision WHL also downloads the PyTorch WHL file
# Read the version from the downloaded whl file without extracting it
PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/')
echo "Found torch release ${PT_RELEASE}"
printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
# Read the commit hash from the downloaded whl file without extracting it
PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'")
echo "Found torch commit hash ${PT_HASH}"
PT_HASH_CHANGED=0
echo "${PT_HASH}" | cmp - pytorch-hash.txt --quiet || PT_HASH_CHANGED=$?
echo "${PT_HASH}" > pytorch-hash.txt
rm torch-"${PT_RELEASE}"*.whl
# Write the release and hash to the environment file so that we can
# retrieve them when creating a PR
echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
echo "PTVISION_RELEASE=${VISION_RELEASE}" >> ${GITHUB_ENV}
echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV}
- name: Build and test (out-of-tree), also update ODS and abstract interpretation library
if: env.PT_HASH_CHANGED != '0'
run: |
cd ${GITHUB_WORKSPACE}
TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \
TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \
TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="ON" \
./build_tools/python_deploy/build_linux_packages.sh
- name: Post issue comment on build failure
if: failure()
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: 1690
body: |
The RollPyTorch action has failed. See [CI log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
The following information may come handy when fixing the code.
```
torch version: ${{ env.PT_RELEASE }}
torch commit hash: ${{ env.PT_HASH }}
torchvision version: ${{ env.PTVISION_RELEASE }}
```
- name: Update PyTorch Build Cache (if running on main branch)
if: github.ref_name == 'main'
id: cache-pytorch
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse
key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }}

- name: Commit changes locally
if: env.PT_HASH_CHANGED != '0'
run: |
cd ${GITHUB_WORKSPACE}
git config user.email "torch-mlir@users.noreply.github.com"
git config user.name "Roll PyTorch Action"
git fetch --recurse-submodules=no
git checkout main
git pull origin main
- name: Create pull request
uses: peter-evans/create-pull-request@v5.0.1
with:
author: Roll PyTorch Action <torch-mlir@users.noreply.github.com>
branch: rollpytorch
body: |
torch version: ${{ env.PT_RELEASE }}
torch commit hash: ${{ env.PT_HASH }}
torchvision version: ${{ env.PTVISION_RELEASE }}
commit-message: |
update PyTorch version to ${{ env.PT_RELEASE }}
- torch version: ${{ env.PT_RELEASE }}
- torch commit hash: ${{ env.PT_HASH }}
- torchvision version: ${{ env.PTVISION_RELEASE }}
committer: Roll PyTorch Action <torch-mlir@users.noreply.github.com>
title: update PyTorch version to ${{ env.PT_RELEASE }}
token: ${{ secrets.ROLLPYTORCH_TOKEN0 }}
- name: Prepare workspace
run: |
# Clear the workspace directory so that we don't run into errors about
# existing lock files.
sudo rm -rf $GITHUB_WORKSPACE/*
- name: Get torch-mlir
uses: actions/checkout@v3
with:
submodules: 'false'
token: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}

- name: Get LLVM and StableHlo submodules
run: |
set -eo pipefail
cd ${GITHUB_WORKSPACE}
# Fetching the submodules concurrently may cause problems, so we fetch
# them one after another.
rm -f .git/modules/externals/llvm-project/index.lock
rm -f .git/modules/externals/stablehlo/index.lock
git submodule update --init --recursive externals/llvm-project
git submodule update --init --recursive externals/stablehlo
- name: Setup ccache
uses: ./.github/actions/setup-build
with:
cache-suffix: 'rollPyTorch'

- name: Determine nightly PyTorch version
run: |
set -eo pipefail
cd ${GITHUB_WORKSPACE}
python -m pip install wheel
sudo apt-get install unzip
# Fetch the most recent nightly torchvision release
VISION_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torchvision | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
echo "Found torchvision release ${VISION_RELEASE}"
# Fetch the whl file associated with the nightly torchvision release
rm -f torch*.whl
python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torchvision==${VISION_RELEASE}"
# Downloading the torchvision WHL also downloads the PyTorch WHL file
# Read the version from the downloaded whl file without extracting it
PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/')
echo "Found torch release ${PT_RELEASE}"
printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt
# Read the commit hash from the downloaded whl file without extracting it
PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | tail -1 | awk '{ print $3 }' | tr -d "'")
echo "Found torch commit hash ${PT_HASH}"
PT_HASH_CHANGED=0
echo "${PT_HASH}" | cmp - pytorch-hash.txt --quiet || PT_HASH_CHANGED=$?
echo "${PT_HASH}" > pytorch-hash.txt
rm torch-"${PT_RELEASE}"*.whl
# Write the release and hash to the environment file so that we can
# retrieve them when creating a PR
echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
echo "PTVISION_RELEASE=${VISION_RELEASE}" >> ${GITHUB_ENV}
echo "PT_HASH_CHANGED=${PT_HASH_CHANGED}" >> ${GITHUB_ENV}
- name: Build and test (out-of-tree), also update ODS and abstract interpretation library
if: env.PT_HASH_CHANGED != '0'
run: |
cd ${GITHUB_WORKSPACE}
TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \
TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \
TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB="ON" \
./build_tools/python_deploy/build_linux_packages.sh
- name: Post issue comment on build failure
if: failure()
uses: peter-evans/create-or-update-comment@v2
with:
issue-number: 1690
body: |
The RollPyTorch action has failed. See [CI log](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
The following information may come handy when fixing the code.
```
torch version: ${{ env.PT_RELEASE }}
torch commit hash: ${{ env.PT_HASH }}
torchvision version: ${{ env.PTVISION_RELEASE }}
```
- name: Update PyTorch Build Cache (if running on main branch)
if: github.ref_name == 'main'
id: cache-pytorch
uses: actions/cache@v3
with:
path: ${{ github.workspace }}/build_tools/python_deploy/wheelhouse
key: ${{ runner.os }}-pytorch-${{ env.PT_HASH }}

- name: Commit changes locally
if: env.PT_HASH_CHANGED != '0'
run: |
cd ${GITHUB_WORKSPACE}
git config user.email "torch-mlir@users.noreply.github.com"
git config user.name "Roll PyTorch Action"
git fetch --recurse-submodules=no
git checkout main
git pull origin main
- name: Create pull request
uses: peter-evans/create-pull-request@v5.0.1
with:
author: Roll PyTorch Action <torch-mlir@users.noreply.github.com>
branch: rollpytorch
body: |
torch version: ${{ env.PT_RELEASE }}
torch commit hash: ${{ env.PT_HASH }}
torchvision version: ${{ env.PTVISION_RELEASE }}
commit-message: |
update PyTorch version to ${{ env.PT_RELEASE }}
- torch version: ${{ env.PT_RELEASE }}
- torch commit hash: ${{ env.PT_HASH }}
- torchvision version: ${{ env.PTVISION_RELEASE }}
committer: Roll PyTorch Action <torch-mlir@users.noreply.github.com>
title: update PyTorch version to ${{ env.PT_RELEASE }}
token: ${{ secrets.ROLLPYTORCH_TOKEN0 }}
Loading

0 comments on commit 4a4d80a

Please sign in to comment.