Skip to content

Commit 74c0dd8

Browse files
lhotarilhotari
authored
[Build] Add GitHub Actions build for pulsar-adapters (#19)
1 parent 238f8a9 commit 74c0dd8

File tree

8 files changed

+338
-59
lines changed

8 files changed

+338
-59
lines changed

‎.github/actions/tune-runner-vm/action.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: Tune Runner VM performance
21+
description: tunes the GitHub Runner VM operation system
22+
runs:
23+
using: composite
24+
steps:
25+
- run: |
26+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
27+
# Ensure that reverse lookups for current hostname are handled properly
28+
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
29+
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
30+
31+
# The default vm.swappiness setting is 60 which has a tendency to start swapping when memory
32+
# consumption is high.
33+
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
34+
echo 1 | sudo tee /proc/sys/vm/swappiness
35+
36+
# use "madvise" Linux Transparent HugePages (THP) setting
37+
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
38+
# "madvise" is generally a better option than the default "always" setting
39+
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
40+
41+
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
42+
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
43+
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
44+
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /
45+
sudo mount -o remount,nodiscard,commit=999999,barrier=0 /mnt
46+
# disable discard/trim at device level since remount with nodiscard doesn't seem to be effective
47+
# https://www.spinics.net/lists/linux-ide/msg52562.html
48+
for i in /sys/block/sd*/queue/discard_max_bytes; do
49+
echo 0 | sudo tee $i
50+
done
51+
# disable any background jobs that run SSD discard/trim
52+
sudo systemctl disable fstrim.timer || true
53+
sudo systemctl stop fstrim.timer || true
54+
sudo systemctl disable fstrim.service || true
55+
sudo systemctl stop fstrim.service || true
56+
fi
57+
shell: bash

‎.github/changes-filter.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# contains pattern definitions used in workflows "changes" step
2+
# pattern syntax: https://github.com/micromatch/picomatch
3+
all:
4+
- '**'
5+
docs:
6+
- 'site2/**'
7+
- 'deployment/**'
8+
- '.asf.yaml'
9+
- '*.md'
10+
- '**/*.md'

‎.github/workflows/ci-maven-cache-update.yaml‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
# This workflow keeps the GitHub Action Caches up-to-date in the repository.
21+
#
22+
# A pull request build cannot update the cache of the upstream repository. Pull
23+
# requests have a cache in the context of the fork of the upstream repository.
24+
# A pull request build will lookup cache entries from the cache of the upstream
25+
# repository in the case that a cache entry is missing in the pull request source
26+
# repository's cache.
27+
# To reduce cache misses for pull request builds, it is necessary that the
28+
# caches in the upstream repository are up-to-date.
29+
# If the cache entry already exists, the cache won't be updated. This will keep the
30+
# update job very efficient and the downloading and updating will only run if one of the pom.xml
31+
# files has been modified or the cache entry expires.
32+
#
33+
34+
name: CI - Maven Dependency Cache Update
35+
on:
36+
# trigger on every commit to given branches
37+
push:
38+
branches:
39+
- master
40+
# trigger on a schedule so that the cache will be rebuilt if it happens to expire
41+
schedule:
42+
- cron: '30 */12 * * *'
43+
44+
env:
45+
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
46+
47+
jobs:
48+
update-maven-dependencies-cache:
49+
name: Update Maven dependency cache for ${{ matrix.name }}
50+
runs-on: ${{ matrix.runs-on }}
51+
timeout-minutes: 45
52+
53+
strategy:
54+
fail-fast: false
55+
matrix:
56+
include:
57+
- name: all modules
58+
runs-on: ubuntu-latest
59+
cache_name: 'm2-dependencies'
60+
mvn_arguments: ''
61+
62+
steps:
63+
- name: checkout
64+
uses: actions/checkout@v2
65+
66+
- name: Tune Runner VM
67+
uses: ./.github/actions/tune-runner-vm
68+
69+
- name: Detect changed files
70+
if: ${{ github.event_name != 'schedule' }}
71+
id: changes
72+
uses: apache/pulsar-test-infra/paths-filter@master
73+
with:
74+
filters: |
75+
poms:
76+
- 'pom.xml'
77+
- '**/pom.xml'
78+
79+
- name: Cache local Maven repository
80+
if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }}
81+
id: cache
82+
uses: actions/cache@v2
83+
with:
84+
path: |
85+
~/.m2/repository/*/*/*
86+
!~/.m2/repository/org/apache/pulsar
87+
key: ${{ runner.os }}-${{ matrix.cache_name }}-${{ hashFiles('**/pom.xml') }}
88+
# there is no restore-keys here so that the cache size doesn't keep
89+
# on growing from old entries which wouldn't never expire if the old
90+
# cache would be used as the starting point for a new cache entry
91+
92+
- name: Set up JDK 11
93+
uses: actions/setup-java@v2
94+
if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
95+
with:
96+
distribution: 'adopt'
97+
java-version: 11
98+
99+
- name: Download dependencies
100+
if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
101+
run: |
102+
# download dependencies, ignore errors
103+
mvn -B -fn -ntp ${{ matrix.mvn_arguments }} dependency:go-offline

‎.github/workflows/integration-test.yaml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: CI - Integration
21+
on:
22+
pull_request:
23+
branches:
24+
- master
25+
push:
26+
branches:
27+
- branch-*
28+
29+
env:
30+
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
31+
32+
jobs:
33+
34+
integration-tests:
35+
name:
36+
runs-on: ubuntu-latest
37+
timeout-minutes: 120
38+
39+
steps:
40+
- name: checkout
41+
uses: actions/checkout@v2
42+
43+
- name: Tune Runner VM
44+
uses: ./.github/actions/tune-runner-vm
45+
46+
- name: Detect changed files
47+
id: changes
48+
uses: apache/pulsar-test-infra/paths-filter@master
49+
with:
50+
filters: .github/changes-filter.yaml
51+
52+
- name: Check changed files
53+
id: check_changes
54+
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
55+
56+
- name: Cache local Maven repository
57+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
58+
uses: actions/cache@v2
59+
with:
60+
path: |
61+
~/.m2/repository/*/*/*
62+
!~/.m2/repository/org/apache/pulsar
63+
key: ${{ runner.os }}-m2-dependencies-${{ hashFiles('**/pom.xml') }}
64+
restore-keys: |
65+
${{ runner.os }}-m2-dependencies-
66+
67+
- name: Set up JDK 11
68+
uses: actions/setup-java@v2
69+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
70+
with:
71+
distribution: 'adopt'
72+
java-version: 11
73+
74+
- name: install org.apache.pulsar.tests:integration:jar:tests:2.8.0
75+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
76+
run: |
77+
cd ~
78+
git clone --depth 50 --single-branch --branch v2.8.0 https://github.com/apache/pulsar
79+
cd pulsar
80+
mvn -B -ntp -f tests/pom.xml -pl org.apache.pulsar.tests:tests-parent,org.apache.pulsar.tests:integration install
81+
82+
- name: build apachepulsar/pulsar-test-latest-version:latest
83+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
84+
run: |
85+
docker pull apachepulsar/pulsar-all:2.8.0
86+
docker pull apachepulsar/pulsar:2.8.0
87+
docker tag apachepulsar/pulsar-all:2.8.0 apachepulsar/pulsar-all:latest
88+
docker tag apachepulsar/pulsar:2.8.0 apachepulsar/pulsar:latest
89+
cd ~/pulsar
90+
mvn -B -ntp -f tests/docker-images/pom.xml install -pl org.apache.pulsar.tests:latest-version-image -am -Pdocker,-main -DskipTests
91+
92+
- name: run integration tests
93+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
94+
run: mvn -B -ntp -pl org.apache.pulsar.tests:pulsar-kafka-compat-client-test,org.apache.pulsar.tests:pulsar-spark-test -am -DintegrationTests -DskipTests -DredirectTestOutputToFile=false clean verify
95+
96+
- name: package surefire artifacts
97+
if: failure()
98+
run: |
99+
rm -rf artifacts
100+
mkdir artifacts
101+
find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
102+
zip -r artifacts.zip artifacts
103+
104+
- uses: actions/upload-artifact@master
105+
name: upload surefire-artifacts
106+
if: failure()
107+
with:
108+
name: surefire-artifacts
109+
path: artifacts.zip

‎.github/workflows/unit-test.yaml‎

Lines changed: 29 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -27,69 +27,57 @@ on:
2727
- branch-*
2828

2929
env:
30-
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=3
30+
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
3131

3232
jobs:
33-
unit-test:
33+
34+
unit-tests:
3435
name:
3536
runs-on: ubuntu-latest
36-
container:
37-
image: apachepulsar/pulsar-build:latest
3837
timeout-minutes: 120
3938

4039
steps:
41-
- name: Cancel Previous Runs
42-
uses: styfle/cancel-workflow-action@0.6.0
43-
with:
44-
access_token: ${{ github.token }}
45-
4640
- name: checkout
4741
uses: actions/checkout@v2
48-
with:
49-
fetch-depth: 25
50-
ref: ${{ github.event.pull_request.head.sha }}
51-
path: adapters
5242

53-
- name: Clone Pulsar
54-
uses: actions/checkout@v2
43+
- name: Tune Runner VM
44+
uses: ./.github/actions/tune-runner-vm
45+
46+
- name: Detect changed files
47+
id: changes
48+
uses: apache/pulsar-test-infra/paths-filter@master
5549
with:
56-
fetch-depth: 1
57-
repository: apache/pulsar
58-
ref: master
59-
path: pulsar
50+
filters: .github/changes-filter.yaml
51+
52+
- name: Check changed files
53+
id: check_changes
54+
run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
6055

6156
- name: Cache local Maven repository
57+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
6258
uses: actions/cache@v2
6359
with:
64-
path: ~/.m2/repository
65-
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
60+
path: |
61+
~/.m2/repository/*/*/*
62+
!~/.m2/repository/org/apache/pulsar
63+
key: ${{ runner.os }}-m2-dependencies-${{ hashFiles('**/pom.xml') }}
6664
restore-keys: |
67-
${{ runner.os }}-maven-
68-
69-
- name: Set up JDK 1.8
70-
uses: actions/setup-java@v1
71-
with:
72-
java-version: 1.8
65+
${{ runner.os }}-m2-dependencies-
7366
74-
- name: Set up Maven
75-
uses: apache/pulsar-test-infra/setup-maven@master
67+
- name: Set up JDK 11
68+
uses: actions/setup-java@v2
69+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
7670
with:
77-
maven-version: 3.6.1
71+
distribution: 'adopt'
72+
java-version: 11
7873

79-
- name: Build pulsar
80-
run: |
81-
cd pulsar
82-
mvn -B -DskipTests -Dcheckstyle.skip install
83-
84-
- name: Build pulsar adapters
85-
run: |
86-
cd adapters
87-
mvn clean license:check install
74+
- name: run unit tests
75+
if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
76+
run: mvn -B -ntp -pl '!org.apache.pulsar.tests:pulsar-kafka-compat-client-test,!org.apache.pulsar.tests:pulsar-spark-test' clean verify
8877

8978
- name: package surefire artifacts
9079
if: failure()
9180
run: |
92-
cd adapters
9381
rm -rf artifacts
9482
mkdir artifacts
9583
find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
@@ -100,4 +88,4 @@ jobs:
10088
if: failure()
10189
with:
10290
name: surefire-artifacts
103-
path: adapters/artifacts.zip
91+
path: artifacts.zip

0 commit comments

Comments
 (0)