[Build] Add GitHub Actions build for pulsar-adapters

.github/actions/tune-runner-vm/action.yml

@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Tune Runner VM performance
+description: tunes the GitHub Runner VM operation system
+runs:
+  using: composite
+  steps:
+    - run: |
+        if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+            # Ensure that reverse lookups for current hostname are handled properly
+            # Add the current IP address, long hostname and short hostname record to /etc/hosts file
+            echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+
+            # The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
+            # consumption is high.
+            # Set vm.swappiness=1 to avoid swapping and allow high RAM usage
+            echo 1 | sudo tee /proc/sys/vm/swappiness
+
+            # use "madvise" Linux Transparent HugePages (THP) setting
+            # https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
+            # "madvise" is generally a better option than the default "always" setting
+            echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
+
+            # tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
+            # commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
+            # nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
+            sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
+            # disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
+            # https://www.spinics.net/lists/linux-ide/msg52562.html
+            for i in /sys/block/sd*/queue/discard_max_bytes; do
+              echo 0 | sudo tee $i
+            done
+            # disable any background jobs that run SSD discard/trim
+            sudo systemctl disable fstrim.timer || true
+            sudo systemctl stop fstrim.timer || true
+            sudo systemctl disable fstrim.service || true
+            sudo systemctl stop fstrim.service || true
+        fi
+      shell: bash

.github/changes-filter.yaml

@@ -0,0 +1,10 @@
+# contains pattern definitions used in workflows "changes" step
+# pattern syntax: https://github.com/micromatch/picomatch
+all:
+  - '**'
+docs:
+  - 'site2/**'
+  - 'deployment/**'
+  - '.asf.yaml'
+  - '*.md'
+  - '**/*.md'

.github/workflows/ci-maven-cache-update.yaml

@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# This workflow keeps the GitHub Action Caches up-to-date in the repository.
+#
+# A pull request build cannot update the cache of the upstream repository. Pull
+# requests have a cache in the context of the fork of the upstream repository.
+# A pull request build will lookup cache entries from the cache of the upstream
+# repository in the case that a cache entry is missing in the pull request source
+# repository's cache.
+# To reduce cache misses for pull request builds, it is necessary that the
+# caches in the upstream repository are up-to-date.
+# If the cache entry already exists, the cache won't be updated. This will keep the
+# update job very efficient and the downloading and updating will only run if one of the pom.xml
+# files has been modified or the cache entry expires.
+#
+
+name: CI - Maven Dependency Cache Update
+on:
+  # trigger on every commit to given branches
+  push:
+    branches:
+      - master
+  # trigger on a schedule so that the cache will be rebuilt if it happens to expire
+  schedule:
+    - cron: '30 */12 * * *'
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+  update-maven-dependencies-cache:
+    name: Update Maven dependency cache for ${{ matrix.name }}
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 45
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: all modules
+            runs-on: ubuntu-latest
+            cache_name: 'm2-dependencies'
+            mvn_arguments: ''
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Detect changed files
+        if: ${{ github.event_name != 'schedule' }}
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: |
+            poms:
+              - 'pom.xml'
+              - '**/pom.xml'
+
+      - name: Cache local Maven repository
+        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }}
+        id: cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/pulsar
+          key: ${{ runner.os }}-${{ matrix.cache_name }}-${{ hashFiles('**/pom.xml') }}
+          # there is no restore-keys here so that the cache size doesn't keep
+          # on growing from old entries which wouldn't never expire if the old
+          # cache would be used as the starting point for a new cache entry
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
+        with:
+          distribution: 'adopt'
+          java-version: 11
+
+      - name: Download dependencies
+        if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
+        run: |
+          # download dependencies, ignore errors
+          mvn -B -fn -ntp ${{ matrix.mvn_arguments }} dependency:go-offline

.github/workflows/integration-test.yaml

@@ -0,0 +1,109 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: CI - Integration
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - branch-*
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+
+  integration-tests:
+    name:
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Detect changed files
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
+        with:
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
+      - name: Cache local Maven repository
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/pulsar
+          key: ${{ runner.os }}-m2-dependencies-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-m2-dependencies-
+
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        with:
+          distribution: 'adopt'
+          java-version: 11
+
+      - name: install org.apache.pulsar.tests:integration:jar:tests:2.8.0
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          cd ~
+          git clone --depth 50 --single-branch --branch v2.8.0  https://github.com/apache/pulsar
+          cd pulsar
+          mvn -B -ntp -f tests/pom.xml -pl org.apache.pulsar.tests:tests-parent,org.apache.pulsar.tests:integration install
+
+      - name: build apachepulsar/pulsar-test-latest-version:latest
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: |
+          docker pull apachepulsar/pulsar-all:2.8.0
+          docker pull apachepulsar/pulsar:2.8.0
+          docker tag apachepulsar/pulsar-all:2.8.0 apachepulsar/pulsar-all:latest
+          docker tag apachepulsar/pulsar:2.8.0 apachepulsar/pulsar:latest
+          cd ~/pulsar
+          mvn -B -ntp -f tests/docker-images/pom.xml install -pl org.apache.pulsar.tests:latest-version-image -am -Pdocker,-main -DskipTests
+
+      - name: run integration tests
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -B -ntp -pl org.apache.pulsar.tests:pulsar-kafka-compat-client-test,org.apache.pulsar.tests:pulsar-spark-test -am -DintegrationTests -DskipTests -DredirectTestOutputToFile=false clean verify
+
+      - name: package surefire artifacts
+        if: failure()
+        run: |
+          rm -rf artifacts
+          mkdir artifacts
+          find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
+          zip -r artifacts.zip artifacts
+
+      - uses: actions/upload-artifact@master
+        name: upload surefire-artifacts
+        if: failure()
+        with:
+          name: surefire-artifacts
+          path: artifacts.zip

.github/workflows/unit-test.yaml

@@ -27,69 +27,57 @@ on:
       - branch-*
 
 env:
-  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
-  unit-test:
+
+  unit-tests:
     name:
     runs-on: ubuntu-latest
-    container:
-      image: apachepulsar/pulsar-build:latest
     timeout-minutes: 120
 
     steps:
-      - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.6.0
-        with:
-          access_token: ${{ github.token }}
-
       - name: checkout
         uses: actions/checkout@v2
-        with:
-          fetch-depth: 25
-          ref: ${{ github.event.pull_request.head.sha }}
-          path: adapters
 
-      - name: Clone Pulsar
-        uses: actions/checkout@v2
+      - name: Tune Runner VM
+        uses: ./.github/actions/tune-runner-vm
+
+      - name: Detect changed files
+        id:   changes
+        uses: apache/pulsar-test-infra/paths-filter@master
         with:
-          fetch-depth: 1
-          repository: apache/pulsar
-          ref: master
-          path: pulsar
+          filters: .github/changes-filter.yaml
+
+      - name: Check changed files
+        id: check_changes
+        run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
 
       - name: Cache local Maven repository
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         uses: actions/cache@v2
         with:
-          path: ~/.m2/repository
-          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          path: |
+            ~/.m2/repository/*/*/*
+            !~/.m2/repository/org/apache/pulsar
+          key: ${{ runner.os }}-m2-dependencies-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            ${{ runner.os }}-maven-
-
-      - name: Set up JDK 1.8
-        uses: actions/setup-java@v1
-        with:
-          java-version: 1.8
+            ${{ runner.os }}-m2-dependencies-
 
-      - name: Set up Maven
-        uses: apache/pulsar-test-infra/setup-maven@master
+      - name: Set up JDK 11
+        uses: actions/setup-java@v2
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
         with:
-          maven-version: 3.6.1
+          distribution: 'adopt'
+          java-version: 11
 
-      - name: Build pulsar
-        run: |
-          cd pulsar
-          mvn -B -DskipTests -Dcheckstyle.skip install
-
-      - name: Build pulsar adapters
-        run: |
-          cd adapters
-          mvn clean license:check install
+      - name: run unit tests
+        if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
+        run: mvn -B -ntp -pl '!org.apache.pulsar.tests:pulsar-kafka-compat-client-test,!org.apache.pulsar.tests:pulsar-spark-test' clean verify
 
       - name: package surefire artifacts
         if: failure()
         run: |
-          cd adapters
           rm -rf artifacts
           mkdir artifacts
           find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
@@ -100,4 +88,4 @@ jobs:
         if: failure()
         with:
           name: surefire-artifacts
-          path: adapters/artifacts.zip
+          path: artifacts.zip