Scalability improvements: Increment artifact version server side #3998
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: ci-slow | |
| on: | |
| workflow_dispatch: | |
| workflow_call: | |
| push: | |
| branches: [main] | |
| paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] | |
| pull_request: | |
| types: [opened, synchronize, ready_for_review] | |
| paths-ignore: [docs/**, docker/**, '*', '!pyproject.toml', '**.md'] | |
| concurrency: | |
| # New commit on branch cancels running workflows of the same branch | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-slow-ci-label-is-set: | |
| runs-on: ubuntu-latest | |
| if: github.event.pull_request.draft == false | |
| steps: | |
| # using this instead of contains(github.event.pull_request.labels.*.name, 'run-slow-ci') | |
| # to make it dynamic, otherwise github context is fixed at the moment of trigger event. | |
| # With dynamic approach dev can set label and rerun this flow to make it running. | |
| - name: Get PR labels | |
| id: pr-labels | |
| uses: actions/github-script@v5 | |
| with: | |
| script: | | |
| const prNumber = ${{ github.event.pull_request.number }}; | |
| const { data: labels } = await github.rest.issues.listLabelsOnIssue({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: prNumber, | |
| }); | |
| const labelNames = labels.map(label => label.name); | |
| core.setOutput('all-labels', labelNames.join(',')); | |
| - name: Slow CI label not set | |
| if: ${{ !contains(steps.pr-labels.outputs.all-labels, 'run-slow-ci') }} | |
| run: | | |
| echo "Please add the 'run-slow-ci' label to this PR before merging." | |
| exit 1 | |
| mysql-db-migration-testing-full: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mysql full | |
| mysql-db-migration-testing-random: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mysql random | |
| sqlite-db-migration-testing-full: | |
| needs: run-slow-ci-label-is-set | |
| runs-on: arc-runner-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| if: github.event.pull_request.draft == false | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9' | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh sqlite full | |
| mariadb-db-migration-testing: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| env: | |
| ZENML_ANALYTICS_OPT_IN: false | |
| ZENML_DEBUG: true | |
| runs-on: arc-runner-set | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.9' | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| if: github.event.pull_request.head.repo.fork == false | |
| - name: Test migrations across versions | |
| run: bash scripts/test-migrations.sh mariadb full | |
| small-checks: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4.1.1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5.0.0 | |
| with: | |
| python-version: '3.11' | |
| - name: Install uv | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| source $HOME/.cargo/env | |
| - name: Create virtual environment | |
| run: | | |
| uv venv | |
| - name: Check for broken dependencies | |
| run: | | |
| source .venv/bin/activate | |
| uv pip check | |
| - name: Markdown link check | |
| uses: gaurav-nelson/github-action-markdown-link-check@1.0.15 | |
| with: | |
| use-quiet-mode: 'yes' | |
| use-verbose-mode: 'no' | |
| folder-path: ./examples, ./docs/book, ./src | |
| file-path: ./README.md, ./LICENSE, ./RELEASE_NOTES.md, CODE-OF-CONDUCT.md, | |
| CONTRIBUTING.md, CLA.md, RELEASE_NOTES.md, ROADMAP.md | |
| config-file: .github/workflows/markdown_check_config.json | |
| continue-on-error: true | |
| - name: Security check | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install bandit | |
| bash scripts/check-security.sh | |
| - name: Check for alembic branch divergence | |
| env: | |
| ZENML_DEBUG: 0 | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install alembic | |
| bash scripts/check-alembic-branches.sh | |
| - name: Install latest dashboard (test gitignore) | |
| run: bash scripts/install-dashboard.sh | |
| custom-ubuntu-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [arc-runner-set] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| # IMPORTANT: Since we are using the following combination | |
| # in our `ci-fast` workflow, this combination has been | |
| # excluded from the `ci-slow` workflow. | |
| exclude: | |
| - os: arc-runner-set | |
| python-version: '3.10' | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| windows-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| macos-unit-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [macos-latest] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) | |
| # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 | |
| exclude: | |
| - {python-version: '3.9', os: macos-latest} | |
| include: | |
| - {python-version: '3.9', os: macos-13} | |
| fail-fast: false | |
| uses: ./.github/workflows/unit-test.yml | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| os: ${{ matrix.os }} | |
| secrets: inherit | |
| windows-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [windows-latest] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| test_environment: [default] | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit | |
| macos-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [macos-13] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| # Python 3.9 is on macos-13 but not macos-latest (macos-14-arm64) | |
| # https://github.com/actions/setup-python/issues/696#issuecomment-1637587760 | |
| exclude: | |
| - {python-version: '3.9', os: macos-latest} | |
| include: | |
| - {python-version: '3.9', os: macos-13} | |
| test_environment: [default] | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit | |
| custom-ubuntu-integration-test: | |
| if: github.event.pull_request.draft == false | |
| needs: run-slow-ci-label-is-set | |
| strategy: | |
| matrix: | |
| os: [arc-runner-set] | |
| python-version: ['3.9', '3.10', '3.11', '3.12'] | |
| test_environment: | |
| - default | |
| - docker-server-docker-orchestrator-mysql | |
| - docker-server-docker-orchestrator-mariadb | |
| exclude: | |
| # docker is time-consuming to run, so we only run it on 3.9 | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.10' | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.11' | |
| - test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.12' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.10' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.11' | |
| - test_environment: docker-server-docker-orchestrator-mariadb | |
| python-version: '3.12' | |
| # IMPORTANT: Since we are using the following combinations | |
| # in our `ci-fast` workflow, this combination has been | |
| # excluded from the `ci-slow` workflow. | |
| - os: arc-runner-set | |
| test_environment: default | |
| python-version: '3.10' | |
| - os: arc-runner-set | |
| test_environment: docker-server-docker-orchestrator-mysql | |
| python-version: '3.10' | |
| fail-fast: false | |
| uses: ./.github/workflows/integration-test-slow.yml | |
| with: | |
| os: ${{ matrix.os }} | |
| python-version: ${{ matrix.python-version }} | |
| test_environment: ${{ matrix.test_environment }} | |
| secrets: inherit |