Merge pull request #382 from Altinity/pull-request-workflow

Pull request workflow
Altinity · Nov 28, 2023 · d0a6c84 · d0a6c84
2 parents 5883bf4 + 848e4c5
commit d0a6c84
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 43 deletions.
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -1,38 +1,55 @@
-name: Sink connector(Kafka version) Docker Build/Push
+name: Kafka - Build
 
-on:
-  pull_request:
-    branches: [ main ]
+on: 
+  workflow_call:
 
 jobs:
-  build:
+  build-kafka:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+
       - name: Set up JDK 11
         uses: actions/setup-java@v2
         with:
           java-version: '11'
           distribution: 'temurin'
           cache: maven
+
+      - name: Setup env
+        working-directory: sink-connector
+        run: |
+          echo "IMAGE_TAG=${{ github.event.number }}-$GITHUB_SHA-kafka" >> "$GITHUB_ENV"
+          echo "PROJECT_VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)" >> "$GITHUB_ENV"
+
       - name: Build with Maven
         working-directory: sink-connector
         run: mvn -B package --file pom.xml
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
+
+      - name: Upload Maven build
+        uses: actions/upload-artifact@v3
+        with:
+          name: clickhouse-sink-connector_${{ env.IMAGE_TAG }}.jar
+          path: sink-connector/target/clickhouse-kafka-sink-connector-${{ env.PROJECT_VERSION }}.jar
+
+      - name: Build with Go
+        working-directory: sink-connector-client
+        run: CGO_ENABLED=0 go build
+
+      - name: Upload sink-connector-client
+        uses: actions/upload-artifact@v3
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-        if: ${{ env.username != '' }}
+          name: clickhouse-sink-connector-client_${{ env.IMAGE_TAG }}
+          path: sink-connector-client/sink-connector-client
+
       - name: Build the Docker image
-        env:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-        if: ${{ env.username != '' }}
         working-directory: sink-connector
-        run: docker build . --file docker/Dockerfile-sink-on-debezium-base-image --tag altinity/clickhouse-sink-connector:$(date +%F)
-      - name: Push docker image to dockerhub
-        env:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-        if: ${{ env.username != '' }}
-        working-directory: sink-connector
-        run: docker push altinity/clickhouse-sink-connector:$(date +%F)
+        run: |
+          docker build . --file docker/Dockerfile-sink-on-debezium-base-image --tag clickhouse-sink-connector:${{ env.IMAGE_TAG }}
+          docker save clickhouse-sink-connector:${{ env.IMAGE_TAG }}| gzip > clickhouse-sink-connector_${{ env.IMAGE_TAG }}.tar.gz
+
+      - name: Upload Docker tar
+        uses: actions/upload-artifact@v3
+        with:
+          name: clickhouse-sink-connector_${{ env.IMAGE_TAG }}.tar.gz
+          path: sink-connector/clickhouse-sink-connector_${{ env.IMAGE_TAG }}.tar.gz
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
@@ -0,0 +1,24 @@
+name: Kafka Sink Connector - Pull Request Pipeline
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - reopened
+      - opened
+    branches:
+      - '**'
+
+jobs:
+  build-kafka:
+    uses: ./.github/workflows/docker-build.yml
+
+  testflows-kafka:
+    needs: [build-kafka]
+    uses: ./.github/workflows/testflows-sink-connector-kafka.yml
+    with:
+      sink_version: ${{ github.event.number }}-${{ github.sha }}-kafka
+
+  java-tests-kafka:
+    needs: [build-kafka]
+    uses: ./.github/workflows/sink-connector-kafka-tests.yml
diff --git a/.github/workflows/sink-connector-kafka-tests.yml b/.github/workflows/sink-connector-kafka-tests.yml
@@ -1,13 +1,10 @@
-name: Sink Connector(Kafka version) tests
+name: Kafka - Java tests
 
-on:
-  push:
-    branches: [ main, develop ]
-  pull_request:
-    branches: [ main, develop ]
+on: 
+  workflow_call:
 
 jobs:
-  build:
+  java-tests:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
@@ -22,4 +19,4 @@ jobs:
       run: mvn -B package --file pom.xml
     - name: Run Unit tests
       working-directory: sink-connector
-      run: mvn test -Dgroups=IntegrationTest --file pom.xml
+      run: mvn test -Dgroups=IntegrationTest --file pom.xml
diff --git a/.github/workflows/sink-connector-lightweight-tests.yml b/.github/workflows/sink-connector-lightweight-tests.yml
@@ -1,13 +1,10 @@
-name: Sink Connector(Light-weight) Tests
+name: Lightweight - Java Tests
 
 on:
-  push:
-    branches: [ main, develop]
-  pull_request:
-    branches: [ main , develop]
+  workflow_call:
 
 jobs:
-  build:
+  java-tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2

diff --git a/...rkflows/testflow-sink-connector-kafka.yml → ...kflows/testflows-sink-connector-kafka.yml b/...rkflows/testflow-sink-connector-kafka.yml → ...kflows/testflows-sink-connector-kafka.yml
@@ -1,10 +1,10 @@
 name: Kafka - TestFlows Tests
 
 on:
-  workflow_dispatch:
+  workflow_call:
     inputs:
       sink_version:
-        description: "Kafka version"
+        description: "Kafka connector version"
         required: true
         type: string
         default: "latest"
@@ -19,6 +19,15 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - uses: actions/download-artifact@v3
+        with:
+          name: clickhouse-sink-connector_${{ env.SINK_CONNECTOR_VERSION }}.tar.gz
+
+      - name: Load Docker image
+        run: |
+          docker load  < clickhouse-sink-connector_${{ env.SINK_CONNECTOR_VERSION }}.tar.gz
+          docker image ls
+
       - name: Runner ssh command
         working-directory: sink-connector/tests/integration
         run: echo "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@$(hostname -I | cut -d ' ' -f 1)"
@@ -59,13 +68,13 @@ jobs:
         run: cat raw.log | tfs report coverage ../requirements/requirements.py | tfs document convert > coverage.html
 
       - name: Upload artifacts to Altinity Test Reports S3 bucket
-        if: always()
         working-directory: sink-connector/tests/integration/logs
         run: aws s3 cp . s3://altinity-test-reports/altinity-sink-connector/testflows/${{ steps.date.outputs.date }}_sink/ --recursive --exclude "*" --include "*.log" --include "*.html"
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           AWS_DEFAULT_REGION: 'eu-west-2'
+        if: ${{ env.AWS_ACCESS_KEY_ID != '' }}
 
       - uses: actions/upload-artifact@v3
         if: always()

diff --git a/...s/testflow-sink-connector-lightweight.yml → .../testflows-sink-connector-lightweight.yml b/...s/testflow-sink-connector-lightweight.yml → .../testflows-sink-connector-lightweight.yml
@@ -1,7 +1,7 @@
 name: Lightweight - TestFlows Tests 
 
 on:    
-  workflow_dispatch:
+  workflow_call:
     inputs:
       sink_version:
         description: "Lightweight version"

diff --git a/build_docker.sh b/build_docker.sh
@@ -8,10 +8,6 @@ cd sink-connector-lightweight
 mvn clean install -DskipTests=true
 today_date=$(date +%F)
 
-cd ..
-cd sink-connector-client
-CGO_ENABLED=0 go build
-
 cd ..
 
 docker login registry.gitlab.com

diff --git a/sink-connector/tests/integration/env/docker-compose.yml b/sink-connector/tests/integration/env/docker-compose.yml
@@ -83,7 +83,7 @@ services:
   sink:
     container_name: sink
     hostname: sink
-    image: "altinity/clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}"
+    image: "clickhouse-sink-connector:${SINK_CONNECTOR_VERSION}"
     restart: "no"
     expose:
       - "8083"