[SPARK-49790] Support HPA template for SparkCluster
#433
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| # Cancel previous PR build and test | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| license-check: | |
| name: "License Check" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Check license header | |
| uses: apache/skywalking-eyes@main | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| config: .github/.licenserc.yaml | |
| build-test: | |
| name: "Build Test CI" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| java-version: [ 17, 21 ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'adopt' | |
| cache: 'gradle' | |
| - name: Build with Gradle | |
| run: | | |
| ./gradlew build | |
| build-image: | |
| name: "Build Operator Image CI" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 17 | |
| distribution: 'adopt' | |
| cache: 'gradle' | |
| - name: Build Operator Image | |
| run: | | |
| ./gradlew buildDockerImage | |
| docker run spark-kubernetes-operator:$(./gradlew properties | grep '^version:' | awk '{print $2}') | |
| k8s-integration-tests: | |
| name: "K8s Integration Tests" | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| kubernetes-version: | |
| - "1.28.0" | |
| - "1.31.0" | |
| mode: | |
| - dynamic | |
| - static | |
| test-group: | |
| - spark-versions | |
| - python | |
| - state-transition | |
| - watched-namespaces | |
| exclude: | |
| - mode: dynamic | |
| test-group: spark-versions | |
| - mode: dynamic | |
| test-group: python | |
| - mode: dynamic | |
| test-group: state-transition | |
| - mode: static | |
| test-group: watched-namespaces | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 21 | |
| distribution: 'adopt' | |
| cache: 'gradle' | |
| - name: Set up Minikube | |
| uses: medyagh/setup-minikube@v0.0.18 | |
| with: | |
| cache: true | |
| kubernetes-version: ${{ matrix.kubernetes-version }} | |
| cpus: 2 | |
| memory: 6144m | |
| - name: Set Up Go | |
| uses: actions/setup-go@v5 | |
| - name: Set Up Chainsaw | |
| run: | | |
| go install github.com/kyverno/chainsaw@latest | |
| - name: Print K8S pods and nodes info | |
| run: | | |
| kubectl get pods -A | |
| kubectl describe node | |
| - name: Run Spark K8s Operator on K8S with Dynamic Configuration Disabled | |
| if: matrix.mode == 'static' | |
| run: | | |
| eval $(minikube docker-env) | |
| ./gradlew buildDockerImage | |
| ./gradlew spark-operator-api:relocateGeneratedCRD | |
| helm install spark-kubernetes-operator --create-namespace -f build-tools/helm/spark-kubernetes-operator/values.yaml build-tools/helm/spark-kubernetes-operator/ | |
| helm test spark-kubernetes-operator | |
| # Use remote host' s docker image | |
| minikube docker-env --unset | |
| - name: Run E2E Test with Dynamic Configuration Disabled | |
| if: matrix.mode == 'static' | |
| run: | | |
| chainsaw test --test-dir ./tests/e2e/${{ matrix.test-group }} --parallel 2 | |
| - name: Run Spark K8s Operator on K8S with Dynamic Configuration Enabled | |
| if: matrix.mode == 'dynamic' | |
| run: | | |
| eval $(minikube docker-env) | |
| ./gradlew buildDockerImage | |
| ./gradlew spark-operator-api:relocateGeneratedCRD | |
| helm install spark-kubernetes-operator --create-namespace -f \ | |
| build-tools/helm/spark-kubernetes-operator/values.yaml -f \ | |
| tests/e2e/helm/dynamic-config-values.yaml \ | |
| build-tools/helm/spark-kubernetes-operator/ | |
| minikube docker-env --unset | |
| - name: Run E2E Test with Dynamic Configuration Enabled | |
| if: matrix.mode == 'dynamic' | |
| run: | | |
| chainsaw test --test-dir ./tests/e2e/${{ matrix.test-group }} --parallel 2 | |
| lint: | |
| name: "Linter and documentation" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up JDK 17 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 17 | |
| distribution: 'adopt' | |
| cache: 'gradle' | |
| - name: Linters | |
| run: | | |
| ./gradlew build -x test | |
| - name: Javadoc Generation | |
| run: | | |
| ./gradlew javadoc | |
| - name: Validate helm chart linting | |
| run: | | |
| helm lint --strict build-tools/helm/spark-kubernetes-operator |