Skip to content

Scalability improvements: Increment artifact version server side #3998

Scalability improvements: Increment artifact version server side

Scalability improvements: Increment artifact version server side #3998

Workflow file for this run

---
name: ci-slow
on:
workflow_dispatch:
workflow_call:
push:
branches: [main]
paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
pull_request:
types: [opened, synchronize, ready_for_review]
paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md']
concurrency:
# New commit on branch cancels running workflows of the same branch
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
run-slow-ci-label-is-set:
runs-on: ubuntu-latest
if: github.event.pull_request.draft == false
steps:
# using this instead of contains(github.event.pull_request.labels.*.name, 'run-slow-ci')
# to make it dynamic, otherwise github context is fixed at the moment of trigger event.
# With dynamic approach dev can set label and rerun this flow to make it running.
- name: Get PR labels
id: pr-labels
uses: actions/github-script@v5
with:
script: |
const prNumber = ${{ github.event.pull_request.number }};
const { data: labels } = await github.rest.issues.listLabelsOnIssue({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
});
const labelNames = labels.map(label => label.name);
core.setOutput('all-labels', labelNames.join(','));
- name: Slow CI label not set
if: ${{ !contains(steps.pr-labels.outputs.all-labels, 'run-slow-ci') }}
run: |
echo "Please add the 'run-slow-ci' label to this PR before merging."
exit 1
mysql-db-migration-testing-full:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
env:
ZENML_ANALYTICS_OPT_IN: false
ZENML_DEBUG: true
runs-on: arc-runner-set
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9'
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
if: github.event.pull_request.head.repo.fork == false
- name: Test migrations across versions
run: bash scripts/test-migrations.sh mysql full
mysql-db-migration-testing-random:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
env:
ZENML_ANALYTICS_OPT_IN: false
ZENML_DEBUG: true
runs-on: arc-runner-set
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9'
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
if: github.event.pull_request.head.repo.fork == false
- name: Test migrations across versions
run: bash scripts/test-migrations.sh mysql random
sqlite-db-migration-testing-full:
needs: run-slow-ci-label-is-set
runs-on: arc-runner-set
env:
ZENML_ANALYTICS_OPT_IN: false
ZENML_DEBUG: true
if: github.event.pull_request.draft == false
steps:
- name: Checkout code
uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9'
- name: Test migrations across versions
run: bash scripts/test-migrations.sh sqlite full
mariadb-db-migration-testing:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
env:
ZENML_ANALYTICS_OPT_IN: false
ZENML_DEBUG: true
runs-on: arc-runner-set
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
- name: Set up Python 3.9
uses: actions/setup-python@v5.0.0
with:
python-version: '3.9'
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
if: github.event.pull_request.head.repo.fork == false
- name: Test migrations across versions
run: bash scripts/test-migrations.sh mariadb full
small-checks:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
- name: Set up Python
uses: actions/setup-python@v5.0.0
with:
python-version: '3.11'
- name: Install uv
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
source $HOME/.cargo/env
- name: Create virtual environment
run: |
uv venv
- name: Check for broken dependencies
run: |
source .venv/bin/activate
uv pip check
- name: Markdown link check
uses: gaurav-nelson/github-action-markdown-link-check@1.0.15
with:
use-quiet-mode: 'yes'
use-verbose-mode: 'no'
folder-path: ./examples, ./docs/book, ./src
file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md,
CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md
config-file: .github/workflows/markdown_check_config.json
continue-on-error: true
- name: Security check
run: |
source .venv/bin/activate
uv pip install bandit
bash scripts/check-security.sh
- name: Check for alembic branch divergence
env:
ZENML_DEBUG: 0
run: |
source .venv/bin/activate
uv pip install alembic
bash scripts/check-alembic-branches.sh
- name: Install latest dashboard (test gitignore)
run: bash scripts/install-dashboard.sh
custom-ubuntu-unit-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [arc-runner-set]
python-version: ['3.9', '3.10', '3.11', '3.12']
# IMPORTANT: Since we are using the following combination
# in our `ci-fast` workflow, this combination has been
# excluded from the `ci-slow` workflow.
exclude:
- os: arc-runner-set
python-version: '3.10'
fail-fast: false
uses: ./.github/workflows/unit-test.yml
with:
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
secrets: inherit
windows-unit-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [windows-latest]
python-version: ['3.9', '3.10', '3.11', '3.12']
fail-fast: false
uses: ./.github/workflows/unit-test.yml
with:
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
secrets: inherit
macos-unit-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [macos-latest]
python-version: ['3.9', '3.10', '3.11', '3.12']
# Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
# https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
exclude:
- {python-version: '3.9', os: macos-latest}
include:
- {python-version: '3.9', os: macos-13}
fail-fast: false
uses: ./.github/workflows/unit-test.yml
with:
python-version: ${{ matrix.python-version }}
os: ${{ matrix.os }}
secrets: inherit
windows-integration-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [windows-latest]
python-version: ['3.9', '3.10', '3.11', '3.12']
test_environment: [default]
fail-fast: false
uses: ./.github/workflows/integration-test-slow.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
test_environment: ${{ matrix.test_environment }}
secrets: inherit
macos-integration-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [macos-13]
python-version: ['3.9', '3.10', '3.11', '3.12']
# Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64)
# https://github.com/actions/setup-python/issues/696#issuecomment-1637587760
exclude:
- {python-version: '3.9', os: macos-latest}
include:
- {python-version: '3.9', os: macos-13}
test_environment: [default]
fail-fast: false
uses: ./.github/workflows/integration-test-slow.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
test_environment: ${{ matrix.test_environment }}
secrets: inherit
custom-ubuntu-integration-test:
if: github.event.pull_request.draft == false
needs: run-slow-ci-label-is-set
strategy:
matrix:
os: [arc-runner-set]
python-version: ['3.9', '3.10', '3.11', '3.12']
test_environment:
- default
- docker-server-docker-orchestrator-mysql
- docker-server-docker-orchestrator-mariadb
exclude:
# docker is time-consuming to run, so we only run it on 3.9
- test_environment: docker-server-docker-orchestrator-mysql
python-version: '3.10'
- test_environment: docker-server-docker-orchestrator-mysql
python-version: '3.11'
- test_environment: docker-server-docker-orchestrator-mysql
python-version: '3.12'
- test_environment: docker-server-docker-orchestrator-mariadb
python-version: '3.10'
- test_environment: docker-server-docker-orchestrator-mariadb
python-version: '3.11'
- test_environment: docker-server-docker-orchestrator-mariadb
python-version: '3.12'
# IMPORTANT: Since we are using the following combinations
# in our `ci-fast` workflow, this combination has been
# excluded from the `ci-slow` workflow.
- os: arc-runner-set
test_environment: default
python-version: '3.10'
- os: arc-runner-set
test_environment: docker-server-docker-orchestrator-mysql
python-version: '3.10'
fail-fast: false
uses: ./.github/workflows/integration-test-slow.yml
with:
os: ${{ matrix.os }}
python-version: ${{ matrix.python-version }}
test_environment: ${{ matrix.test_environment }}
secrets: inherit