ci.yml

name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

env:
  CARGO_TERM_COLOR: always

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  check:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - run: sudo snap install --edge --classic just
      - uses: actions/checkout@v4

      - name: Cargo format
        run: just check-format

      - name: Check diff
        run: git diff --exit-code

  docker:
    runs-on: ubuntu-24.04
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v4

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      # - name: Set up Buildx
      #   uses: docker/setup-buildx-action@v3
      #   with:
      #     version: v0.13.1

      - name: Docker info
        run: docker info
      
      - name: Build Docker image (amd64)
        run: |
          DOCKER_BUILDKIT=1 docker build -t localhost/iceberg-rest-local:amd64 \
            -f docker/full.Dockerfile \
            --add-host=host.docker.internal:host-gateway \
            --build-arg DATABASE_URL=postgres://postgres:postgres@host.docker.internal:5432/postgres .
          docker save -o /tmp/iceberg-rest-server-amd64.tar localhost/iceberg-rest-local:amd64
      
      # This is currently very slow. Switch to ARM workers once they are finally available
      # in gh actions!
      # - name: Build Docker image (arm64)
      #   run: |
      #     DOCKER_BUILDKIT=1 docker build \
      #       --platform linux/arm64 -t iceberg-rest-local:arm64 \
      #       --add-host=host.docker.internal:host-gateway \
      #       -f docker/full.Dockerfile \
      #       --build-arg DATABASE_URL=postgres://postgres:postgres@host.docker.internal:5432/postgres .
      #     docker save -o /tmp/iceberg-rest-server-arm64.tar iceberg-rest-local:arm64

      # ToDo: Use action instead of manual build.
      # The snippet below couldn't resolve the host.docker.internal / host-gateway.
      # - name: Build Docker image (amd64)
      #   uses: docker/build-push-action@v5
      #   id: build-amd64
      #   with:
      #     context: .
      #     load: true
      #     platforms: linux/amd64
      #     file: docker/full.Dockerfile
      #     outputs: type=docker,dest=/tmp/iceberg-rest-server-amd64.tar
      #     # cache-from: type=gha
      #     # cache-to: type=gha,mode=max
      #     add-hosts: "host.docker.internal:host-gateway"
      #     build-args: |
      #       "DATABASE_URL=postgres://postgres:postgres@host.docker.internal:5432/postgres"
      #     tags: iceberg-rest-local:amd64

      - name: Save Docker
        uses: actions/upload-artifact@v4
        with:
          name: iceberg-rest-server-image
          path: /tmp/*.tar

  clippy:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - run: sudo snap install --edge --classic just
      - uses: actions/checkout@v4

      - name: Migrate database
        run: |
          sudo apt-get install libpq-dev -y
          cargo install --version=0.7.4 sqlx-cli --no-default-features --features postgres
          cd crates/iceberg-rest-server
          sqlx database create
          sqlx migrate run
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres

      - name: Cargo clippy
        run: just check-clippy
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres

  test:
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:15
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: postgres
          POSTGRES_DB: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v4

      - name: Migrate database
        run: |
          sudo apt-get install libpq-dev -y
          cargo install --version=0.7.4 sqlx-cli --no-default-features --features postgres
          cd crates/iceberg-rest-server
          sqlx database create
          sqlx migrate run
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres

      - name: Test
        run: cargo test --no-fail-fast --all-targets --all-features --workspace
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
      
      - name: Doc Test
        run: cargo test --no-fail-fast --doc --all-features --workspace
        env:
          DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres

  test-pyiceberg:
    needs: docker
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Restore binary
        uses: actions/download-artifact@v4
        with:
          name: iceberg-rest-server-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts
    
      - name: Restore Docker image
        run: |
          docker load -i artifacts/iceberg-rest-server-amd64.tar

      - name: Test Pyiceberg
        run: |
          cd tests &&
          docker compose run spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run_pyiceberg.sh"

        env:
          ICEBERG_REST_TEST_SPARK_IMAGE: apache/spark:3.5.1-java17-python3
          ICEBERG_REST_TEST_SERVER_IMAGE: localhost/iceberg-rest-local:amd64


  test-pyspark:
    needs: docker
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Restore binary
        uses: actions/download-artifact@v4
        with:
          name: iceberg-rest-server-image
          path: artifacts

      - name: Display structure of downloaded files
        run: ls -Rlh artifacts
    
      - name: Restore Docker image
        run: |
          docker load -i artifacts/iceberg-rest-server-amd64.tar

      - name: Test Pyspark
        run: |
          cd tests &&
          docker compose run spark /opt/entrypoint.sh bash -c "cd /opt/tests && bash run_spark.sh"

        env:
          ICEBERG_REST_TEST_SPARK_IMAGE: apache/spark:3.5.1-java17-python3
          ICEBERG_REST_TEST_SERVER_IMAGE: localhost/iceberg-rest-local:amd64