build_and_test.yml

name: Build and test

on:
  push:
    branches:
    - main
  pull_request:
    branches:
    - main

# Cancel previous PR build and test
concurrency:
  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.number || github.sha }}
  cancel-in-progress: true

jobs:
  license-check:
    name: "License Check"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Check license header
        uses: apache/skywalking-eyes@main
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          config: .github/.licenserc.yaml
  build-test:
    name: "Build Test CI"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        java-version: [ 17, 21 ]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
      - name: Set up JDK ${{ matrix.java-version }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.java-version }}
          distribution: 'adopt'
          cache: 'gradle'
      - name: Build with Gradle
        run: |
          ./gradlew build
  build-image:
    name: "Build Operator Image CI"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          java-version: 17
          distribution: 'adopt'
          cache: 'gradle'
      - name: Build Operator Image
        run: |
          ./gradlew buildDockerImage
          docker run spark-kubernetes-operator:$(./gradlew properties | grep '^version:' | awk '{print $2}')
  k8s-integration-tests:
    name: "K8s Integration Tests"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        kubernetes_versions:
          - "1.27.0"
          - "1.31.0"
        test_group:
          - spark-versions
          - python
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up JDK 21
        uses: actions/setup-java@v4
        with:
          java-version: 21
          distribution: 'adopt'
          cache: 'gradle'
      - name: Set up Minikube
        uses: medyagh/setup-minikube@v0.0.18
        with:
          cache: true
          kubernetes-version: ${{ matrix.kube-version }}
          cpus: 2
          memory: 6144m
      - name: Set Up Go
        uses: actions/setup-go@v5
      - name: Set Up Chainsaw
        run: |
          go install github.com/kyverno/chainsaw@latest
      - name: Print K8S pods and nodes info
        run: |
          kubectl get pods -A
          kubectl describe node
      - name: Run Spark K8s Operator on K8S
        run: |
          kubectl create clusterrolebinding serviceaccounts-cluster-admin --clusterrole=cluster-admin --group=system:serviceaccounts || true
          eval $(minikube docker-env)
          ./gradlew buildDockerImage
          ./gradlew spark-operator-api:relocateGeneratedCRD
          helm install spark-kubernetes-operator --create-namespace -f build-tools/helm/spark-kubernetes-operator/values.yaml build-tools/helm/spark-kubernetes-operator/
          helm test spark-kubernetes-operator
          # Use remote host' s docker image
          minikube docker-env --unset
      - name: Run E2E Test
        run: |
          chainsaw test --test-dir ./tests/e2e/${{ matrix.test_group }} --parallel 2
  lint:
    name: "Linter and documentation"
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up JDK 17
        uses: actions/setup-java@v4
        with:
          java-version: 17
          distribution: 'adopt'
          cache: 'gradle'
      - name: Linters
        run: |
          ./gradlew build -x test
      - name: Javadoc Generation
        run: |
          ./gradlew javadoc
      - name: Validate helm chart linting
        run: |
          helm lint --strict build-tools/helm/spark-kubernetes-operator