diff --git a/.asf.yaml b/.asf.yaml
index 6d6ef3fbf39ec3..b4fd3919dc0579 100644
--- a/.asf.yaml
+++ b/.asf.yaml
@@ -49,27 +49,34 @@ github:
# Contexts are the names of checks that must pass.
# See ./github/workflows/README.md for more documentation on this list.
contexts:
- - License check
- - backwards-compatibility
- build
- - cli
- cpp-tests
- - function-state
- - messaging
- - process
- - pulsar-ci-test (group1)
- - pulsar-ci-test (group2)
- - pulsar-ci-test (group3)
- - pulsar-io
- - schema
- - shade-check
- - sql
- - standalone
- - thread
- - tiered-filesystem
- - tiered-jcloud
- - transaction
- - unit-tests
+ - Changed files check
+ - Build and License check
+ - CI - Unit - Brokers - Broker Group 1
+ - CI - Unit - Brokers - Broker Group 2
+ - CI - Unit - Brokers - Broker Group 3
+ - CI - Unit - Brokers - Client Api
+ - CI - Unit - Brokers - Client Impl
+ - CI - Unit - Brokers - JDK 8
+ - CI - Unit - Other
+ - CI - Unit - Proxy
+ - Build Pulsar java-test-image docker image
+ - CI - Integration - Backwards Compatibility
+ - CI - Integration - Cli
+ - CI - Integration - Function
+ - CI - Integration - Messaging
+ - CI - Integration - Schema
+ - CI - Integration - Shade
+ - CI - Integration - Standalone
+ - CI - Integration - Transaction
+ - Build Pulsar docker image
+ - CI - System - Pulsar Connectors - Process
+ - CI - System - Pulsar Connectors - Thread
+ - CI - System - Pulsar IO
+ - CI - System - Sql
+ - CI - System - Tiered FileSystem
+ - CI - System - Tiered JCloud
required_pull_request_reviews:
dismiss_stale_reviews: false
diff --git a/.github/actions/clean-disk/action.yml b/.github/actions/clean-disk/action.yml
new file mode 100644
index 00000000000000..8bcc5f13968026
--- /dev/null
+++ b/.github/actions/clean-disk/action.yml
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: clean disk
+description: makes some more space available on the disk by removing files
+inputs:
+ mode:
+ description: "Use 'full' to clean as much as possible"
+ required: false
+runs:
+ using: composite
+ steps:
+ - run: |
+ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+ directories=(/usr/local/lib/android /opt/ghc)
+ if [[ "${{ inputs.mode }}" == "full" ]]; then
+ # remove these directories only when mode is 'full'
+ directories+=(/usr/share/dotnet)
+ fi
+ emptydir=/tmp/empty$$/
+ mkdir $emptydir
+ echo "::group::Available diskspace"
+ time df -BM / /mnt
+ echo "::endgroup::"
+ for directory in "${directories[@]}"; do
+ echo "::group::Removing $directory"
+ # fast way to delete a lot of files on linux
+ time sudo eatmydata rsync -a --delete $emptydir ${directory}/
+ time sudo eatmydata rm -rf ${directory}
+ time df -BM / /mnt
+ echo "::endgroup::"
+ done
+ echo "::group::Cleaning apt state"
+ time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove"
+ time df -BM / /mnt
+ echo "::endgroup::"
+ fi
+ echo "::group::Available diskspace"
+ time df -BM / /mnt
+ echo "::endgroup::"
+ shell: bash
diff --git a/.github/actions/copy-test-reports/action.yml b/.github/actions/copy-test-reports/action.yml
new file mode 100644
index 00000000000000..7c49afacc60dc8
--- /dev/null
+++ b/.github/actions/copy-test-reports/action.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Copy test reports
+description: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+runs:
+ using: composite
+ steps:
+ - run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh move_test_reports
+ shell: bash
diff --git a/.github/actions/merge-test-reports/action.yml b/.github/actions/merge-test-reports/action.yml
new file mode 100644
index 00000000000000..154e834383c455
--- /dev/null
+++ b/.github/actions/merge-test-reports/action.yml
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Merge test reports to a single file
+description: Merges junit test reports to a single file
+inputs:
+ summary_title:
+ description: 'The title for the summary'
+ required: false
+runs:
+ using: composite
+ steps:
+ - run: |
+ if ls test-reports/*.xml 2>/dev/null; then
+ echo '::group::Installing tooling...'
+ # this requires "runs-on: ubuntu-20.04" since junitparser package is not available in previous Ubuntu version
+ sudo apt-get -y install xmlstarlet junitparser >/dev/null
+ echo '::endgroup::'
+ echo '::group::Parsing and merging test reports...'
+ junitparser merge test-reports/*.xml merged-test-report.xml || true
+ echo '::endgroup::'
+ if [[ -f merged-test-report.xml && -n "${{ inputs.summary_title }}" ]]; then
+ xmlstarlet sel -T -t -m /testsuites -o "::warning::${{ inputs.summary_title }} " -v '@tests' -o ' tests, ' -v '@failures' -o ' failures, ' -v '@errors' -o ' errors, ' -v '@skipped' -o ' skipped, duration ' -v 'number(@time)' -o 's' merged-test-report.xml
+ fi
+ else
+ echo 'No test-reports/*.xml files found.'
+ fi
+ shell: bash
diff --git a/.github/actions/ssh-access/action.yml b/.github/actions/ssh-access/action.yml
new file mode 100644
index 00000000000000..582d5b174409b0
--- /dev/null
+++ b/.github/actions/ssh-access/action.yml
@@ -0,0 +1,147 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: ssh access
+description: Sets up SSH access to build VM with upterm
+inputs:
+ action:
+ description: |
+ Action to perform: options are "start" and "wait"
+ "start" will install, configure and start upterm.
+ "wait" will wait until a connection is established to upterm and will continue to wait until the session is closed.
+ required: false
+ default: 'start'
+ limit-access-to-actor:
+ description: 'If only the public SSH keys of the user triggering the workflow should be authorized'
+ required: false
+ default: 'false'
+ limit-access-to-users:
+ description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.'
+ required: false
+ default: ''
+ secure-access:
+ description: |
+ Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset.
+ required: false
+ default: 'true'
+ timeout:
+ description: 'When action=wait, the timeout in seconds to wait for the user to connect'
+ required: false
+ default: '300'
+runs:
+ using: composite
+ steps:
+ - run: |
+ if [[ "${{ inputs.action }}" == "start" ]]; then
+ echo "::group::Installing upterm & tmux"
+ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+ # install upterm
+ curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm
+
+ # install tmux if it's not present
+ if ! command -v tmux &>/dev/null; then
+ sudo apt-get -y install tmux
+ fi
+ elif [[ "$OSTYPE" == "darwin"* ]]; then
+ brew install owenthereal/upterm/upterm
+ # install tmux if it's not present
+ if ! command -v tmux &>/dev/null; then
+ brew install tmux
+ fi
+ else
+ echo "Unsupported $OSTYPE"
+ exit 0
+ fi
+ echo '::endgroup::'
+ echo "::group::Configuring ssh and ssh keys"
+ # generate ssh key
+ mkdir -p ~/.ssh
+ chmod 0700 ~/.ssh
+ if [ ! -f ~/.ssh/id_rsa ]; then
+ ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa
+ fi
+ if [ ! -f ~/.ssh/id_ed25519 ]; then
+ ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519
+ fi
+ # configure ssh
+ echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config
+ # Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev
+ ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true
+ # @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key
+ cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts
+ authorizedKeysParameter=""
+ authorizedKeysFile=${HOME}/.ssh/authorized_keys
+ if [[ "${{ inputs.secure-access }}" != "false" ]]; then
+ ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$
+ cat /tmp/dummykey$$.pub >> $authorizedKeysFile
+ rm /tmp/dummykey$$ /tmp/dummykey$$.pub
+ fi
+ limit_access_to_actor="${{ inputs.limit-access-to-actor }}"
+ if [[ "${limit_access_to_actor}" == "true" ]]; then
+ echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)"
+ curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile
+ fi
+ limit_access_to_users="${{ inputs.limit-access-to-users }}"
+ for github_user in ${limit_access_to_users//,/ }; do
+ if [[ -n "${github_user}" ]]; then
+ echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)"
+ curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile
+ fi
+ done
+ if [ -f $authorizedKeysFile ]; then
+ chmod 0600 $authorizedKeysFile
+ authorizedKeysParameter="-a $authorizedKeysFile"
+ echo -e "Using $authorizedKeysFile\nContent:\n---------------------------"
+ cat $authorizedKeysFile
+ echo "---------------------------"
+ fi
+ echo '::endgroup::'
+ echo "::group::Starting terminal session and connecting to server"
+ tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43"
+ sleep 2
+ tmux send-keys -t upterm-wrapper q C-m
+ sleep 1
+ tmux set -t upterm-wrapper window-size largest
+ tmux set -t upterm window-size largest
+ echo '::endgroup::'
+ echo -e "\nSSH connection information"
+ shopt -s nullglob
+ upterm session current --admin-socket ~/.upterm/*.sock
+ elif [[ "${{ inputs.action }}" == "wait" ]]; then
+ # only wait if upterm was installed
+ if command -v upterm &>/dev/null; then
+ shopt -s nullglob
+ echo "SSH connection information"
+ upterm session current --admin-socket ~/.upterm/*.sock || {
+ echo "upterm isn't running. Not waiting any longer."
+ exit 0
+ }
+ timeout=${{ inputs.timeout }}
+ echo "Waiting $timeout seconds..."
+ sleep $timeout
+ echo "Keep waiting as long as there's a connected session"
+ while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do
+ sleep 30
+ done
+ echo "No session is connected. Not waiting any longer."
+ else
+ echo "upterm isn't installed"
+ fi
+ fi
+ shell: bash
diff --git a/.github/actions/tune-runner-vm/action.yml b/.github/actions/tune-runner-vm/action.yml
index e8914dbe74f6c7..1b567944ea99fc 100644
--- a/.github/actions/tune-runner-vm/action.yml
+++ b/.github/actions/tune-runner-vm/action.yml
@@ -24,6 +24,7 @@ runs:
steps:
- run: |
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
+ echo "::group::Configure and tune OS"
# Ensure that reverse lookups for current hostname are handled properly
# Add the current IP address, long hostname and short hostname record to /etc/hosts file
echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
@@ -32,18 +33,23 @@ runs:
# consumption is high.
# Set vm.swappiness=1 to avoid swapping and allow high RAM usage
echo 1 | sudo tee /proc/sys/vm/swappiness
- # Set swappiness to 1 for all cgroups and sub-groups
- for swappiness_dir in /sys/fs/cgroup/memory/*/ /sys/fs/cgroup/memory/*/*/; do
- if [ -d "swappiness_dir" ]; then
- echo 1 | sudo tee $(swappiness_dir)memory.swappiness > /dev/null
- fi
- done
+ (
+ shopt -s nullglob
+ # Set swappiness to 1 for all cgroups and sub-groups
+ for swappiness_file in /sys/fs/cgroup/memory/*/memory.swappiness /sys/fs/cgroup/memory/*/*/memory.swappiness; do
+ echo 1 | sudo tee $swappiness_file > /dev/null
+ done
+ ) || true
# use "madvise" Linux Transparent HugePages (THP) setting
# https://www.kernel.org/doc/html/latest/admin-guide/mm/transhuge.html
# "madvise" is generally a better option than the default "always" setting
+ # Based on Azul instructions from https://docs.azul.com/prime/Enable-Huge-Pages#transparent-huge-pages-thp
echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
-
+ echo advise | sudo tee /sys/kernel/mm/transparent_hugepage/shmem_enabled
+ echo defer+madvise | sudo tee /sys/kernel/mm/transparent_hugepage/defrag
+ echo 1 | sudo tee /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
+
# tune filesystem mount options, https://www.kernel.org/doc/Documentation/filesystems/ext4.txt
# commit=999999, effectively disables automatic syncing to disk (default is every 5 seconds)
# nobarrier/barrier=0, loosen data consistency on system crash (no negative impact to empheral CI nodes)
@@ -70,13 +76,27 @@ runs:
# stop Azure Linux agent to save RAM
sudo systemctl stop walinuxagent.service || true
+
+ # enable docker experimental mode which is
+ # required for using "docker build --squash" / "-Ddockerfile.build.squash=true"
+ daemon_json="$(sudo cat /etc/docker/daemon.json | jq '.experimental = true')"
+ echo "$daemon_json" | sudo tee /etc/docker/daemon.json
+ # restart docker daemon
+ sudo systemctl restart docker
+ echo '::endgroup::'
# show memory
+ echo "::group::Available Memory"
free -m
+ echo '::endgroup::'
# show disk
- df -h
+ echo "::group::Available diskspace"
+ df -BM
+ echo "::endgroup::"
# show cggroup
- echo "/actions_job cgroup settings:"
- sudo cgget actions_job
+ echo "::group::Cgroup settings for current cgroup $CURRENT_CGGROUP"
+ CURRENT_CGGROUP=$(cat /proc/self/cgroup | grep '0::' | awk -F: '{ print $3 }')
+ sudo cgget -a $CURRENT_CGGROUP || true
+ echo '::endgroup::'
fi
shell: bash
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index dfefaa7cbc6c04..8fa947ea1f9dae 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -49,4 +49,12 @@ do FILE=$f yq eval -o j '.jobs | to_entries | {"file": env(FILE),"id":.[].key, "
done
```
-Duplicate names are allowed, and all checks with the same name will be treated the same (required or not required).
\ No newline at end of file
+Duplicate names are allowed, and all checks with the same name will be treated the same (required or not required).
+
+When working on workflow changes, one way to find out the names of the status checks is to retrieve the names
+from the PR build run. The "check-runs" can be found by commit id. Here's an example:
+
+```shell
+curl -s "https://api.github.com/repos/apache/pulsar/commits/$(git rev-parse HEAD)/check-runs" | \
+ jq -r '.check_runs | .[] | .name' |sort
+```
diff --git a/.github/workflows/ci-build-macos.yaml b/.github/workflows/ci-build-macos.yaml
index b5f984f15a4aa2..7fc95bcad0556b 100644
--- a/.github/workflows/ci-build-macos.yaml
+++ b/.github/workflows/ci-build-macos.yaml
@@ -26,6 +26,10 @@ on:
branches:
- branch-*
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
diff --git a/.github/workflows/ci-cancel-duplicate-workflows.yaml b/.github/workflows/ci-cancel-duplicate-workflows.yaml
index 9beee04d107c6b..5ddaee46d6eeee 100644
--- a/.github/workflows/ci-cancel-duplicate-workflows.yaml
+++ b/.github/workflows/ci-cancel-duplicate-workflows.yaml
@@ -49,141 +49,15 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: allDuplicates
workflowFileName: ci-go-functions-test.yaml
- - name: cancel duplicate ci-integration-backwards-compatibility.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-backwards-compatibility.yaml
- - name: cancel duplicate ci-integration-cli.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-cli.yaml
- - name: cancel duplicate ci-integration-function-state.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-function-state.yaml
- - name: cancel duplicate ci-integration-messaging.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-messaging.yaml
- - name: cancel duplicate ci-integration-process.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-process.yaml
- - name: cancel duplicate ci-integration-schema.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-schema.yaml
- - name: cancel duplicate ci-integration-sql.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-sql.yaml
- - name: cancel duplicate ci-integration-standalone.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-standalone.yaml
- - name: cancel duplicate ci-integration-thread.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-thread.yaml
- - name: cancel duplicate
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-tiered-filesystem.yaml
- - name: cancel duplicate
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-tiered-filesystem.yaml
- - name: cancel duplicate ci-license.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-license.yaml
- - name: cancel duplicate ci-integration-tiered-jcloud.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-tiered-jcloud.yaml
- - name: cancel duplicate ci-integration-transaction.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-integration-transaction.yaml
- name: cancel duplicate ci-pulsar-website-build.yaml
uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: allDuplicates
workflowFileName: ci-pulsar-website-build.yaml
- - name: cancel duplicate ci-shade-test.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-shade-test.yaml
- - name: cancel duplicate ci-unit.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit.yaml
- - name: cancel duplicate ci-unit-broker-broker-gp1.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit-broker-broker-gp1.yaml
- - name: cancel duplicate ci-unit-broker-broker-gp2.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit-broker-broker-gp2.yaml
- - name: cancel duplicate ci-unit-broker-client-api.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit-broker-client-api.yaml
- - name: cancel duplicate ci-unit-broker-client-impl.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit-broker-client-impl.yaml
- - name: cancel duplicate ci-unit-broker-other.yaml
- uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
- with:
- token: ${{ secrets.GITHUB_TOKEN }}
- cancelMode: allDuplicates
- workflowFileName: ci-unit-broker-other.yaml
- - name: cancel duplicate ci-unit-proxy.yaml
+ - name: cancel duplicate pulsar-ci.yaml
uses: potiuk/cancel-workflow-runs@953e057dc81d3458935a18d1184c386b0f6b5738
with:
token: ${{ secrets.GITHUB_TOKEN }}
cancelMode: allDuplicates
- workflowFileName: ci-unit-proxy.yaml
+ workflowFileName: pulsar-ci.yaml
diff --git a/.github/workflows/ci-cpp-build-centos7.yaml b/.github/workflows/ci-cpp-build-centos7.yaml
index 13c5aa084483aa..5eb0ce02df831d 100644
--- a/.github/workflows/ci-cpp-build-centos7.yaml
+++ b/.github/workflows/ci-cpp-build-centos7.yaml
@@ -31,6 +31,11 @@ on:
paths:
- '.github/workflows/**'
- 'pulsar-client-cpp/**'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
cpp-build-centos7:
diff --git a/.github/workflows/ci-cpp-build-windows.yaml b/.github/workflows/ci-cpp-build-windows.yaml
index d12f5e675b9116..a8a800e39d4298 100644
--- a/.github/workflows/ci-cpp-build-windows.yaml
+++ b/.github/workflows/ci-cpp-build-windows.yaml
@@ -32,6 +32,10 @@ on:
- '.github/workflows/**'
- 'pulsar-client-cpp/**'
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
env:
VCPKG_FEATURE_FLAGS: manifests
diff --git a/.github/workflows/ci-cpp.yaml b/.github/workflows/ci-cpp.yaml
index 24e7c1782264ac..5e355872900e69 100644
--- a/.github/workflows/ci-cpp.yaml
+++ b/.github/workflows/ci-cpp.yaml
@@ -26,6 +26,10 @@ on:
branches:
- branch-*
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
diff --git a/.github/workflows/ci-go-functions-style.yaml b/.github/workflows/ci-go-functions-style.yaml
index 6fe797e4d930ef..34e2716b620166 100644
--- a/.github/workflows/ci-go-functions-style.yaml
+++ b/.github/workflows/ci-go-functions-style.yaml
@@ -32,6 +32,10 @@ on:
- '.github/workflows/**'
- 'pulsar-function-go/**'
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
diff --git a/.github/workflows/ci-go-functions-test.yaml b/.github/workflows/ci-go-functions-test.yaml
index b8ec3268b5b266..dc06dba442dfde 100644
--- a/.github/workflows/ci-go-functions-test.yaml
+++ b/.github/workflows/ci-go-functions-test.yaml
@@ -32,6 +32,10 @@ on:
- '.github/workflows/**'
- 'pulsar-function-go/**'
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
diff --git a/.github/workflows/ci-integration-backwards-compatibility.yaml b/.github/workflows/ci-integration-backwards-compatibility.yaml
deleted file mode 100644
index a4eb8eaef85f4a..00000000000000
--- a/.github/workflows/ci-integration-backwards-compatibility.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Backwards Compatibility
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- backwards-compatibility:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh BACKWARDS_COMPAT
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-cli.yaml b/.github/workflows/ci-integration-cli.yaml
deleted file mode 100644
index a374e9c937ea6b..00000000000000
--- a/.github/workflows/ci-integration-cli.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Cli
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- cli:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh CLI
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-function.yaml b/.github/workflows/ci-integration-function.yaml
deleted file mode 100644
index 4ada7bb22f3d2e..00000000000000
--- a/.github/workflows/ci-integration-function.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Function & IO
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- function-state:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh FUNCTION
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-messaging.yaml b/.github/workflows/ci-integration-messaging.yaml
deleted file mode 100644
index f515bb9941a17a..00000000000000
--- a/.github/workflows/ci-integration-messaging.yaml
+++ /dev/null
@@ -1,119 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Messaging
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- messaging:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh MESSAGING
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-process.yaml b/.github/workflows/ci-integration-process.yaml
deleted file mode 100644
index 8a00e6fb2a0a26..00000000000000
--- a/.github/workflows/ci-integration-process.yaml
+++ /dev/null
@@ -1,123 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Process
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- process:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_PROCESS
-
- - name: Log dmesg when failed
- if: ${{ failure() }}
- continue-on-error: true
- run: sudo dmesg
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-pulsar-io-ora.yaml b/.github/workflows/ci-integration-pulsar-io-ora.yaml
deleted file mode 100644
index e23dde7e39b9da..00000000000000
--- a/.github/workflows/ci-integration-pulsar-io-ora.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Pulsar-IO Oracle Source
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- pulsar-io:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo swapoff -a
- sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO_ORA
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-pulsar-io.yaml b/.github/workflows/ci-integration-pulsar-io.yaml
deleted file mode 100644
index 145623b9279187..00000000000000
--- a/.github/workflows/ci-integration-pulsar-io.yaml
+++ /dev/null
@@ -1,120 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Pulsar-IO Sinks and Sources
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- pulsar-io:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo swapoff -a
- sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_IO
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-schema.yaml b/.github/workflows/ci-integration-schema.yaml
deleted file mode 100644
index f07711a001245c..00000000000000
--- a/.github/workflows/ci-integration-schema.yaml
+++ /dev/null
@@ -1,115 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Schema
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- schema:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SCHEMA
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-sql.yaml b/.github/workflows/ci-integration-sql.yaml
deleted file mode 100644
index 54198d14dfc08d..00000000000000
--- a/.github/workflows/ci-integration-sql.yaml
+++ /dev/null
@@ -1,124 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Sql
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Xmx768m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
- MALLOC_ARENA_MAX: "1"
-
-jobs:
-
- sql:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker pulsar latest test image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SQL
-
- - name: Log dmesg when failed
- if: ${{ failure() }}
- continue-on-error: true
- run: sudo dmesg
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-standalone.yaml b/.github/workflows/ci-integration-standalone.yaml
deleted file mode 100644
index 3f2dde1fa467ef..00000000000000
--- a/.github/workflows/ci-integration-standalone.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Standalone
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- standalone:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh STANDALONE
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-thread.yaml b/.github/workflows/ci-integration-thread.yaml
deleted file mode 100644
index 162a2b15260668..00000000000000
--- a/.github/workflows/ci-integration-thread.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Thread
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- thread:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration function
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh PULSAR_CONNECTORS_THREAD
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-tiered-filesystem.yaml b/.github/workflows/ci-integration-tiered-filesystem.yaml
deleted file mode 100644
index d777a07d93ceda..00000000000000
--- a/.github/workflows/ci-integration-tiered-filesystem.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Tiered FileSystem
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- tiered-filesystem:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_FILESYSTEM
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-tiered-jcloud.yaml b/.github/workflows/ci-integration-tiered-jcloud.yaml
deleted file mode 100644
index 02aa003880368f..00000000000000
--- a/.github/workflows/ci-integration-tiered-jcloud.yaml
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Tiered JCloud
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- tiered-jcloud:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TIERED_JCLOUD
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-integration-transaction.yaml b/.github/workflows/ci-integration-transaction.yaml
deleted file mode 100644
index 39230e4c920870..00000000000000
--- a/.github/workflows/ci-integration-transaction.yaml
+++ /dev/null
@@ -1,115 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Integration - Transaction
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- transaction:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
- - name: build artifacts and docker image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run integration tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh TRANSACTION
-
- - name: Upload container logs
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: container-logs
- path: tests/integration/target/container-logs
-
- - name: Upload surefire-reports
- uses: actions/upload-artifact@v2
- if: ${{ cancelled() || failure() }}
- continue-on-error: true
- with:
- name: surefire-reports
- path: tests/integration/target/surefire-reports
diff --git a/.github/workflows/ci-license.yaml b/.github/workflows/ci-license.yaml
deleted file mode 100644
index ebc9ba09edcb1a..00000000000000
--- a/.github/workflows/ci-license.yaml
+++ /dev/null
@@ -1,90 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Misc
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- license-check:
- name: License check
- runs-on: ubuntu-latest
- timeout-minutes: 60
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
-
- # license check fails with 3.6.2 so we have to downgrade
- - name: Set up Maven
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: apache/pulsar-test-infra/setup-maven@master
- with:
- maven-version: 3.6.1
-
- - name: build and check license
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp -DskipTests apache-rat:check initialize license:check install
-
- - name: license check
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz
diff --git a/.github/workflows/ci-owasp-dep-check.yaml b/.github/workflows/ci-owasp-dep-check.yaml
index b12f9791f33750..a6ab4ff0ee8e70 100644
--- a/.github/workflows/ci-owasp-dep-check.yaml
+++ b/.github/workflows/ci-owasp-dep-check.yaml
@@ -29,6 +29,10 @@ on:
env:
MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
owasp-dep-check:
diff --git a/.github/workflows/ci-python-build-3.9-client.yaml b/.github/workflows/ci-python-build-3.9-client.yaml
index 37fbd4f2e837b4..a1ef4a73d7b234 100644
--- a/.github/workflows/ci-python-build-3.9-client.yaml
+++ b/.github/workflows/ci-python-build-3.9-client.yaml
@@ -31,6 +31,11 @@ on:
paths:
- '.github/workflows/**'
- 'pulsar-client-cpp/**'
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
build-wheel:
diff --git a/.github/workflows/ci-shade-test.yaml b/.github/workflows/ci-shade-test.yaml
deleted file mode 100644
index ffdbeed02b7a5b..00000000000000
--- a/.github/workflows/ci-shade-test.yaml
+++ /dev/null
@@ -1,103 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Shade - Test
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
- UBUNTU_MIRROR: http://azure.archive.ubuntu.com/ubuntu/
-
-jobs:
-
- shade-check:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: clean disk
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: |
- sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
- sudo apt clean
- docker rmi $(docker images -q) -f
- df -h
-
- - name: run install by skip tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -q -B -ntp clean install -DskipTests
-
- - name: build pulsar image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build pulsar-all image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f docker/pulsar-all/pom.xml install -am -Pdocker,-main -DskipTests -Ddocker.nocache=true
-
- - name: build artifacts and docker pulsar latest test image
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -f tests/docker-images/pom.xml install -am -Pdocker,-main -DskipTests
-
- - name: run shade tests
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_integration_group.sh SHADE
diff --git a/.github/workflows/ci-unit-broker-broker-gp.yaml b/.github/workflows/ci-unit-broker-broker-gp.yaml
deleted file mode 100644
index cfe488ba9c3ae3..00000000000000
--- a/.github/workflows/ci-unit-broker-broker-gp.yaml
+++ /dev/null
@@ -1,112 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Brokers - Broker Group
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name: pulsar-ci-test
- runs-on: ubuntu-latest
- timeout-minutes: 60
- strategy:
- fail-fast: false
- matrix:
- group: ["group1", "group2", "group3"]
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: build modules
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'BROKER_GROUP_1'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group1' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_1
-
- - name: run unit test 'BROKER_GROUP_2'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group2' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_2
-
- - name: run unit test 'BROKER_GROUP_3'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' && matrix.group == 'group3' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_GROUP_3
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit-broker-client-api.yaml b/.github/workflows/ci-unit-broker-client-api.yaml
deleted file mode 100644
index e609c2e7e04ad4..00000000000000
--- a/.github/workflows/ci-unit-broker-client-api.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Brokers - Client Api
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: build modules
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'BROKER_CLIENT_API'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_API
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit-broker-client-impl.yaml b/.github/workflows/ci-unit-broker-client-impl.yaml
deleted file mode 100644
index 127deec4fa396b..00000000000000
--- a/.github/workflows/ci-unit-broker-client-impl.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Brokers - Client Impl
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: build modules
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'BROKER_CLIENT_IMPL'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_CLIENT_IMPL
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit-broker-jdk8.yaml b/.github/workflows/ci-unit-broker-jdk8.yaml
deleted file mode 100644
index 3bbc2eaafb4487..00000000000000
--- a/.github/workflows/ci-unit-broker-jdk8.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Broker - JDK8
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 60
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 8
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 8
-
- - name: build modules
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'BROKER_GROUP_1'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_JDK8
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit-broker-other.yaml b/.github/workflows/ci-unit-broker-other.yaml
deleted file mode 100644
index 646e81da6a6986..00000000000000
--- a/.github/workflows/ci-unit-broker-other.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Brokers - Other
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: build modules
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q clean install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'BROKER_FLAKY'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh BROKER_FLAKY
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit-proxy.yaml b/.github/workflows/ci-unit-proxy.yaml
deleted file mode 100644
index 786f793a26c883..00000000000000
--- a/.github/workflows/ci-unit-proxy.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit - Proxy
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: build modules pulsar-proxy
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: mvn -B -ntp -q install -Pcore-modules,-main -DskipTests
-
- - name: run unit test 'PROXY'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh PROXY
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/ci-unit.yaml b/.github/workflows/ci-unit.yaml
deleted file mode 100644
index b488c8198adc0f..00000000000000
--- a/.github/workflows/ci-unit.yaml
+++ /dev/null
@@ -1,96 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: CI - Unit
-on:
- pull_request:
- branches:
- - master
- push:
- branches:
- - branch-*
-
-env:
- MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-
-jobs:
-
- unit-tests:
- name:
- runs-on: ubuntu-latest
- timeout-minutes: 120
-
- steps:
- - name: checkout
- uses: actions/checkout@v2
-
- - name: Tune Runner VM
- uses: ./.github/actions/tune-runner-vm
-
- - name: Detect changed files
- id: changes
- uses: apache/pulsar-test-infra/paths-filter@master
- with:
- filters: .github/changes-filter.yaml
- list-files: csv
-
- - name: Check changed files
- id: check_changes
- run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
-
- - name: Cache local Maven repository
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- uses: actions/cache@v2
- with:
- path: |
- ~/.m2/repository/*/*/*
- !~/.m2/repository/org/apache/pulsar
- key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
- restore-keys: |
- ${{ runner.os }}-m2-dependencies-core-modules-
-
- - name: Set up JDK 11
- uses: actions/setup-java@v2
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- with:
- distribution: 'temurin'
- java-version: 11
-
- - name: run unit test 'OTHER'
- if: ${{ steps.check_changes.outputs.docs_only != 'true' }}
- run: CHANGED_TESTS="${{ steps.changes.outputs.tests_files }}" ./build/run_unit_group.sh OTHER
-
- - name: print JVM thread dumps when cancelled
- if: cancelled()
- run: ./build/pulsar_ci_tool.sh print_thread_dumps
-
- - name: package surefire artifacts
- if: failure()
- run: |
- rm -rf artifacts
- mkdir artifacts
- find . -type d -name "*surefire*" -exec cp --parents -R {} artifacts/ \;
- zip -r artifacts.zip artifacts
-
- - uses: actions/upload-artifact@master
- name: upload surefire-artifacts
- if: failure()
- with:
- name: surefire-artifacts
- path: artifacts.zip
diff --git a/.github/workflows/pulsar-ci-test-report.yaml b/.github/workflows/pulsar-ci-test-report.yaml
new file mode 100644
index 00000000000000..28094e5154ccf3
--- /dev/null
+++ b/.github/workflows/pulsar-ci-test-report.yaml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: 'Pulsar CI Test Report'
+on:
+ workflow_run:
+ workflows: ['Pulsar CI'] # runs after Pulsar CI workflow
+ types:
+ - completed
+jobs:
+ report:
+ runs-on: ubuntu-20.04
+ steps:
+ - uses: apache/pulsar-test-infra/test-reporter@master
+ with:
+ artifact: /(.*)-test-report/
+ name: '$1 Tests'
+ path: '*.xml'
+ reporter: java-junit
+ max-annotations: 50
+ token: ${{ secrets.GITHUB_TOKEN }}
+ fail-on-error: false
diff --git a/.github/workflows/pulsar-ci.yaml b/.github/workflows/pulsar-ci.yaml
new file mode 100644
index 00000000000000..e2259fbd4f6cd9
--- /dev/null
+++ b/.github/workflows/pulsar-ci.yaml
@@ -0,0 +1,666 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: Pulsar CI
+on:
+ pull_request:
+ branches:
+ - master
+ push:
+ branches:
+ - branch-*
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ MAVEN_OPTS: -Dmaven.test.failure.ignore=true -DtestRetryCount=0 -Xmx1024m -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+
+jobs:
+ changed_files_job:
+ name: 'Changed files check'
+ runs-on: ubuntu-20.04
+ outputs:
+ docs_only: ${{ steps.check_changes.outputs.docs_only }}
+ changed_tests: ${{ steps.changes.outputs.tests_files }}
+ steps:
+ - name: checkout
+ uses: actions/checkout@v2
+
+ - name: Detect changed files
+ id: changes
+ uses: apache/pulsar-test-infra/paths-filter@master
+ with:
+ filters: .github/changes-filter.yaml
+ list-files: csv
+
+ - name: Check changed files
+ id: check_changes
+ run: echo "::set-output name=docs_only::${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}"
+
+ build-and-license-check:
+ needs: changed_files_job
+ name: Build and License check
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache local Maven repository
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: 11
+
+ - name: Check source code license headers
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: mvn -B -T 8 -ntp initialize apache-rat:check license:check
+
+ - name: Build core-modules
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true
+
+ - name: Check binary licenses
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: Save maven build results to Github artifact cache so that the results can be reused
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
+ tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
+ -I zstd -cf - .m2/repository/org/apache/pulsar
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ unit-tests:
+ name: CI - Unit - ${{ matrix.name }}
+ runs-on: ubuntu-20.04
+ timeout-minutes: ${{ matrix.timeout || 60 }}
+ needs: ['changed_files_job', 'build-and-license-check']
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Other
+ group: OTHER
+ timeout: 75
+ - name: Brokers - Broker Group 1
+ group: BROKER_GROUP_1
+ - name: Brokers - Broker Group 2
+ group: BROKER_GROUP_2
+ - name: Brokers - Broker Group 3
+ group: BROKER_GROUP_3
+ - name: Brokers - JDK 8
+ group: BROKER_JDK8
+ jdk: 8
+ - name: Brokers - Client Api
+ group: BROKER_CLIENT_API
+ - name: Brokers - Client Impl
+ group: BROKER_CLIENT_IMPL
+ - name: Brokers - Flaky
+ group: BROKER_FLAKY
+ timeout: 75
+ - name: Proxy
+ group: PROXY
+
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache Maven dependencies
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK ${{ matrix.jdk || '11' }}
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: ${{ matrix.jdk || '11' }}
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Run setup commands
+ if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ ${{ matrix.setup }}
+
+ - name: Run unit test group '${{ matrix.group }}'
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ CHANGED_TESTS="${{ needs.changed_files_job.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}
+
+ - name: print JVM thread dumps when cancelled
+ if: cancelled()
+ run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps
+
+ - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/copy-test-reports
+
+ - name: Report test summary as warning
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/merge-test-reports
+ with:
+ summary_title: 'Test Summary for Unit - ${{ matrix.name }}:'
+
+ - name: Publish the Test reports in Junit xml format
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: Unit-${{ matrix.group }}-test-report
+ path: test-reports
+
+ - name: Upload Surefire reports
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: Unit-${{ matrix.group }}-surefire-reports
+ path: surefire-reports
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ pulsar-java-test-image:
+ name: Build Pulsar java-test-image docker image
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ needs: ['changed_files_job', 'build-and-license-check']
+ env:
+ UBUNTU_MIRROR: mirror://mirrors.ubuntu.com/mirrors.txt
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache Maven dependencies
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: 11
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Build java-test-image docker image
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ # build docker image
+ mvn -B -am -pl tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
+ -Dmaven.test.skip=true -Ddockerfile.build.squash=true -DskipSourceReleaseAssembly=true \
+ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
+
+ - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ integration-tests:
+ name: CI - Integration - ${{ matrix.name }}
+ runs-on: ubuntu-20.04
+ timeout-minutes: ${{ matrix.timeout || 60 }}
+ needs: ['changed_files_job', 'pulsar-java-test-image']
+ env:
+ PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Backwards Compatibility
+ group: BACKWARDS_COMPAT
+
+ - name: Cli
+ group: CLI
+
+ - name: Function
+ group: FUNCTION
+
+ - name: Messaging
+ group: MESSAGING
+
+ - name: Schema
+ group: SCHEMA
+
+ - name: Shade
+ group: SHADE
+
+ - name: Standalone
+ group: STANDALONE
+
+ - name: Transaction
+ group: TRANSACTION
+
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache Maven dependencies
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: 11
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image
+
+ - name: Run setup commands
+ if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ ${{ matrix.setup }}
+
+ - name: Run integration test group '${{ matrix.group }}'
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ ./build/run_integration_group.sh ${{ matrix.group }}
+
+ - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/copy-test-reports
+
+ - name: Report test summary as warning
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/merge-test-reports
+ with:
+ summary_title: 'Test Summary for Integration - ${{ matrix.name }}:'
+
+ - name: Publish the Test reports in Junit xml format
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: Integration-${{ matrix.group }}-test-report
+ path: test-reports
+
+ - name: Upload Surefire reports
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: Integration-${{ matrix.group }}-surefire-reports
+ path: surefire-reports
+
+ - name: Upload container logs
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ continue-on-error: true
+ with:
+ name: Integration-${{ matrix.group }}-container-logs
+ path: tests/integration/target/container-logs
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ pulsar-test-latest-version-image:
+ name: Build Pulsar docker image
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ needs: ['changed_files_job', 'build-and-license-check']
+ env:
+ UBUNTU_MIRROR: mirror://mirrors.ubuntu.com/mirrors.txt
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Clean Disk
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/clean-disk
+
+ - name: Cache local Maven repository
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: 11
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Build latest-version-image docker image
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ # build docker image
+ # include building of Pulsar SQL, Connectors, Offloaders and server distros
+ mvn -B -am -pl pulsar-sql/presto-distribution,distribution/io,distribution/offloaders,distribution/server,tests/docker-images/latest-version-image install \
+ -Pmain,docker -Dmaven.test.skip=true -Ddockerfile.build.squash=true \
+ -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true
+
+ - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
+
+ system-tests:
+ name: CI - System - ${{ matrix.name }}
+ runs-on: ubuntu-20.04
+ timeout-minutes: 60
+ needs: ['changed_files_job', 'pulsar-test-latest-version-image']
+ env:
+ PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: Tiered FileSystem
+ group: TIERED_FILESYSTEM
+
+ - name: Tiered JCloud
+ group: TIERED_JCLOUD
+
+ - name: Pulsar Connectors - Thread
+ group: PULSAR_CONNECTORS_THREAD
+
+ - name: Pulsar Connectors - Process
+ group: PULSAR_CONNECTORS_PROCESS
+
+ - name: Pulsar IO
+ group: PULSAR_IO
+
+ - name: Pulsar IO - Oracle
+ group: PULSAR_IO_ORA
+
+ - name: Sql
+ group: SQL
+
+ steps:
+ - name: checkout
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/checkout@v2
+
+ - name: Tune Runner VM
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/tune-runner-vm
+
+ - name: Setup ssh access to build runner VM
+ # ssh access is enabled for builds in own forks
+ if: ${{ github.repository != 'apache/pulsar' && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/ssh-access
+ with:
+ limit-access-to-actor: true
+
+ - name: Cache local Maven repository
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.m2/repository/*/*/*
+ !~/.m2/repository/org/apache/pulsar
+ key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
+ restore-keys: |
+ ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
+ ${{ runner.os }}-m2-dependencies-core-modules-
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v2
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ distribution: 'temurin'
+ java-version: 11
+
+ - name: Install gh-actions-artifact-client.js
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: lhotari/gh-actions-artifact-client/dist@master
+
+ - name: Restore maven build results from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ cd $HOME
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
+
+ - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image
+
+ - name: Run setup commands
+ if: ${{ matrix.setup && needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ ${{ matrix.setup }}
+
+ - name: Run system test group '${{ matrix.group }}'
+ if: ${{ needs.changed_files_job.outputs.docs_only != 'true' }}
+ run: |
+ ./build/run_integration_group.sh ${{ matrix.group }}
+
+ - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/copy-test-reports
+
+ - name: Report test summary as warning
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ uses: ./.github/actions/merge-test-reports
+ with:
+ summary_title: 'Test Summary for System - ${{ matrix.name }}:'
+
+ - name: Publish the Test reports in Junit xml format
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: System-${{ matrix.group }}-test-report
+ path: test-reports
+
+ - name: Upload container logs
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ continue-on-error: true
+ with:
+ name: System-${{ matrix.group }}-container-logs
+ path: tests/integration/target/container-logs
+
+ - name: Upload Surefire reports
+ uses: actions/upload-artifact@v2
+ if: ${{ always() && needs.changed_files_job.outputs.docs_only != 'true' }}
+ with:
+ name: System-${{ matrix.name }}-surefire-reports
+ path: surefire-reports
+
+ - name: Wait for ssh connection when build fails
+ # ssh access is enabled for builds in own forks
+ uses: ./.github/actions/ssh-access
+ if: ${{ failure() && github.repository != 'apache/pulsar' }}
+ continue-on-error: true
+ with:
+ action: wait
diff --git a/build/pulsar_ci_tool.sh b/build/pulsar_ci_tool.sh
index 988a715cdc53d2..f8270eb5367bff 100755
--- a/build/pulsar_ci_tool.sh
+++ b/build/pulsar_ci_tool.sh
@@ -22,7 +22,7 @@
# lists all available functions in this tool
function ci_list_functions() {
- declare -F | awk '{print $NF}' | sort | egrep '^ci_' | sed 's/^ci_//'
+ declare -F | awk '{print $NF}' | sort | grep -E '^ci_' | sed 's/^ci_//'
}
# prints thread dumps for all running JVMs
@@ -37,6 +37,7 @@ function ci_print_thread_dumps() {
return 0
}
+# runs maven
function _ci_mvn() {
mvn -B -ntp "$@"
}
@@ -46,6 +47,121 @@ function ci_dependency_check() {
_ci_mvn -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl '!pulsar-client-tools-test' "$@"
}
+# installs a tool executable if it's not found on the PATH
+function ci_install_tool() {
+ local tool_executable=$1
+ local tool_package=${2:-$1}
+ if ! command -v $tool_executable &>/dev/null; then
+ echo "::group::Installing ${tool_package}"
+ sudo apt-get -y install ${tool_package} >/dev/null
+ echo '::endgroup::'
+ fi
+}
+
+# outputs the given message to stderr and exits the shell script
+function fail() {
+ echo "$*" >&2
+ exit 1
+}
+
+# function to retry a given commend 3 times with a backoff of 10 seconds in between
+function ci_retry() {
+ local n=1
+ local max=3
+ local delay=10
+ while true; do
+ "$@" && break || {
+ if [[ $n -lt $max ]]; then
+ ((n++))
+ echo "::warning::Command failed. Attempt $n/$max:"
+ sleep $delay
+ else
+ fail "::error::The command has failed after $n attempts."
+ fi
+ }
+ done
+}
+
+# saves a given image (1st parameter) to the GitHub Actions Artifacts with the given name (2nd parameter)
+function ci_docker_save_image_to_github_actions_artifacts() {
+ local image=$1
+ local artifactname="${2}.zst"
+ ci_install_tool pv
+ echo "::group::Saving docker image ${image} with name ${artifactname} in GitHub Actions Artifacts"
+ docker save ${image} | zstd | pv -ft -i 5 | pv -Wbaf -i 5 | gh-actions-artifact-client.js upload "${artifactname}"
+ echo "::endgroup::"
+}
+
+# loads a docker image from the GitHub Actions Artifacts with the given name (1st parameter)
+function ci_docker_load_image_from_github_actions_artifacts() {
+ local artifactname="${1}.zst"
+ ci_install_tool pv
+ echo "::group::Loading docker image from name ${artifactname} in GitHub Actions Artifacts"
+ gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | unzstd | docker load
+ echo "::endgroup::"
+}
+
+# loads and extracts a zstd (.tar.zst) compressed tar file from the GitHub Actions Artifacts with the given name (1st parameter)
+function ci_restore_tar_from_github_actions_artifacts() {
+ local artifactname="${1}.tar.zst"
+ ci_install_tool pv
+ echo "::group::Restoring tar from name ${artifactname} in GitHub Actions Artifacts to $PWD"
+ gh-actions-artifact-client.js download "${artifactname}" | pv -batf -i 5 | tar -I zstd -xf -
+ echo "::endgroup::"
+}
+
+# stores a given command (with full arguments, specified after 1st parameter) output to GitHub Actions Artifacts with the given name (1st parameter)
+function ci_store_tar_to_github_actions_artifacts() {
+ local artifactname="${1}.tar.zst"
+ shift
+ ci_install_tool pv
+ echo "::group::Storing $1 tar command output to name ${artifactname} in GitHub Actions Artifacts"
+ "$@" | pv -ft -i 5 | pv -Wbaf -i 5 | gh-actions-artifact-client.js upload "${artifactname}"
+ echo "::endgroup::"
+}
+
+# copies test reports into test-reports and surefire-reports directory
+# subsequent runs of tests might overwrite previous reports. This ensures that all test runs get reported.
+function ci_move_test_reports() {
+ (
+ if [ -n "${GITHUB_WORKSPACE}" ]; then
+ cd "${GITHUB_WORKSPACE}"
+ mkdir -p test-reports
+ mkdir -p surefire-reports
+ fi
+ # aggregate all junit xml reports in a single directory
+ if [ -d test-reports ]; then
+ # copy test reports to single directory, rename duplicates
+ find . -path '*/target/surefire-reports/junitreports/TEST-*.xml' -print0 | xargs -0 -r -n 1 mv -t test-reports --backup=numbered
+ # rename possible duplicates to have ".xml" extension
+ (
+ for f in test-reports/*~; do
+ mv -- "$f" "${f}.xml"
+ done 2>/dev/null
+ ) || true
+ fi
+ # aggregate all surefire-reports in a single directory
+ if [ -d surefire-reports ]; then
+ (
+ find . -type d -path '*/target/surefire-reports' -not -path './surefire-reports/*' |
+ while IFS=$'\n' read -r directory; do
+ echo "Copying reports from $directory"
+ target_dir="surefire-reports/${directory}"
+ if [ -d "$target_dir" ]; then
+ # rotate backup directory names *~3 -> *~2, *~2 -> *~3, *~1 -> *~2, ...
+ ( command ls -vr1d "${target_dir}~"* 2> /dev/null | awk '{print "mv "$0" "substr($0,0,length-1)substr($0,length,1)+1}' | sh ) || true
+ # backup existing target directory, these are the results of the previous test run
+ mv "$target_dir" "${target_dir}~1"
+ fi
+ # copy files
+ cp -R --parents "$directory" surefire-reports
+ # remove the original directory
+ rm -rf "$directory"
+ done
+ )
+ fi
+ )
+}
if [ -z "$1" ]; then
echo "usage: $0 [ci_tool_function_name]"
@@ -56,11 +172,11 @@ fi
ci_function_name="ci_$1"
shift
-if [[ "$(LC_ALL=C type -t $ci_function_name)" == "function" ]]; then
+if [[ "$(LC_ALL=C type -t "${ci_function_name}")" == "function" ]]; then
eval "$ci_function_name" "$@"
else
echo "Invalid ci tool function"
echo "Available ci tool functions:"
ci_list_functions
exit 1
-fi
\ No newline at end of file
+fi
diff --git a/build/retry.sh b/build/retry.sh
index 6a00d52d2bf33e..4982e3bf29e8e7 100755
--- a/build/retry.sh
+++ b/build/retry.sh
@@ -38,10 +38,10 @@ function retry {
fi
fi
((n++))
- echo "Command failed. Attempt $n/$max:"
+ echo "::warning::Command failed. Attempt $n/$max:"
sleep $delay;
else
- fail "The command has failed after $n attempts."
+ fail "::error::The command has failed after $n attempts."
fi
}
done
diff --git a/build/run_integration_group.sh b/build/run_integration_group.sh
index 2c86da854a6c40..630a08ea8367f4 100755
--- a/build/run_integration_group.sh
+++ b/build/run_integration_group.sh
@@ -24,20 +24,30 @@ set -e
set -o pipefail
set -o errexit
-TEST_GROUP=$1
-if [ -z "$TEST_GROUP" ]; then
- echo "usage: $0 [test_group]"
- exit 1
-fi
-shift
+# lists all active maven modules with given parameters
+# parses the modules from the "mvn initialize" output
+# returns a CSV value
+mvn_list_modules() {
+ (
+ mvn -B -ntp "$@" initialize \
+ | grep -- "-< .* >-" \
+ | sed -E 's/.*-< (.*) >-.*/\1/' \
+ | tr '\n' ',' | sed 's/,$/\n/'
+ )
+}
# runs integration tests
+# 1. cds to "tests" directory and lists the active modules to be used as value
+# for "-pl" parameter of later mvn commands
+# 2. runs "mvn -pl [active_modules] -am install [given_params]" to build and install required dependencies
+# 3. finally runs tests with "mvn -pl [active_modules] test [given_params]"
mvn_run_integration_test() {
(
+ set +x
RETRY=""
# wrap with retry.sh script if next parameter is "--retry"
if [[ "$1" == "--retry" ]]; then
- RETRY="./build/retry.sh"
+ RETRY="${SCRIPT_DIR}/retry.sh"
shift
fi
# skip wrapping with retry.sh script if next parameter is "--no-retry"
@@ -45,10 +55,26 @@ mvn_run_integration_test() {
RETRY=""
shift
fi
+ skip_build_deps=0
+ while [[ "$1" == "--skip-build-deps" ]]; do
+ skip_build_deps=1
+ shift
+ done
+ cd "$SCRIPT_DIR"/../tests
+ modules=$(mvn_list_modules -DskipDocker "$@")
+ cd ..
set -x
-
- # run the integration tests
- $RETRY mvn -B -ntp -DredirectTestOutputToFile=false -f tests/pom.xml test "$@"
+ if [ $skip_build_deps -ne 1 ]; then
+ echo "::group::Build dependencies for $modules"
+ mvn -B -T 1C -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dmaven.test.skip=true -Dcheckstyle.skip=true -Drat.skip=true -am install "$@"
+ echo "::endgroup::"
+ fi
+ echo "::group::Run tests for " "$@"
+ # use "verify" instead of "test"
+ $RETRY mvn -B -ntp -pl "$modules" -DskipDocker -DskipSourceReleaseAssembly=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DredirectTestOutputToFile=false verify "$@"
+ echo "::endgroup::"
+ set +x
+ "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
)
}
@@ -66,7 +92,7 @@ test_group_cli() {
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-cli.xml -DintegrationTests
# run pulsar auth integration tests
- mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-auth.xml -DintegrationTests
+ mvn_run_integration_test --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-auth.xml -DintegrationTests
}
test_group_function() {
@@ -77,9 +103,9 @@ test_group_messaging() {
# run integration messaging tests
mvn_run_integration_test "$@" -DintegrationTestSuiteFile=pulsar-messaging.xml -DintegrationTests
# run integration proxy tests
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-proxy.xml -DintegrationTests
# run integration proxy with WebSocket tests
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-proxy-websocket.xml -DintegrationTests
}
test_group_schema() {
@@ -107,10 +133,10 @@ test_group_pulsar_connectors_thread() {
mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=function
# run integration source
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=source
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=source
# run integration sink
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=sink
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-thread.xml -DintegrationTests -Dgroups=sink
}
test_group_pulsar_connectors_process() {
@@ -118,10 +144,10 @@ test_group_pulsar_connectors_process() {
mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=function
# run integration source
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=source
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=source
# run integraion sink
- mvn_run_integration_test --retry "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=sink
+ mvn_run_integration_test --retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-process.xml -DintegrationTests -Dgroups=sink
}
test_group_sql() {
@@ -130,20 +156,33 @@ test_group_sql() {
test_group_pulsar_io() {
mvn_run_integration_test --no-retry "$@" -DintegrationTestSuiteFile=pulsar-io-sources.xml -DintegrationTests -Dgroups=source
- mvn_run_integration_test --no-retry "$@" -DintegrationTestSuiteFile=pulsar-io-sinks.xml -DintegrationTests -Dgroups=sink
+ mvn_run_integration_test --no-retry --skip-build-deps "$@" -DintegrationTestSuiteFile=pulsar-io-sinks.xml -DintegrationTests -Dgroups=sink
}
test_group_pulsar_io_ora() {
mvn_run_integration_test --no-retry "$@" -DintegrationTestSuiteFile=pulsar-io-ora-source.xml -DintegrationTests -Dgroups=source -DtestRetryCount=0
}
+list_test_groups() {
+ declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
+}
+TEST_GROUP=$1
+if [ -z "$TEST_GROUP" ]; then
+ echo "usage: $0 [test_group]"
+ echo "Available test groups:"
+ list_test_groups
+ exit 1
+fi
+shift
echo "Test Group : $TEST_GROUP"
test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')"
-if [[ "$(LC_ALL=C type -t $test_group_function_name)" == "function" ]]; then
+if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then
eval "$test_group_function_name" "$@"
else
echo "INVALID TEST GROUP"
+ echo "Available test groups:"
+ list_test_groups
exit 1
fi
diff --git a/build/run_unit_group.sh b/build/run_unit_group.sh
index ca5b887780a2d1..3414a5fb141e85 100755
--- a/build/run_unit_group.sh
+++ b/build/run_unit_group.sh
@@ -18,10 +18,37 @@
# under the License.
#
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+
set -e
set -o pipefail
set -o errexit
+MVN_TEST_OPTIONS='mvn -B -ntp -DskipSourceReleaseAssembly=true -DskipBuildDistribution=true -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true'
+
+function mvn_test() {
+ (
+ RETRY="${SCRIPT_DIR}/retry.sh"
+ # skip wrapping with retry.sh script if next parameter is "--no-retry"
+ if [[ "$1" == "--no-retry" ]]; then
+ RETRY=""
+ shift
+ fi
+ TARGET=verify
+ if [[ "$1" == "--install" ]]; then
+ TARGET="install"
+ shift
+ fi
+ echo "::group::Run tests for " "$@"
+ # use "verify" instead of "test" to workaround MDEP-187 issue in pulsar-functions-worker and pulsar-broker projects with the maven-dependency-plugin's copy goal
+ # Error message was "Artifact has not been packaged yet. When used on reactor artifact, copy should be executed after packaging: see MDEP-187"
+ $RETRY $MVN_TEST_OPTIONS $TARGET "$@" "${COMMANDLINE_ARGS[@]}"
+ echo "::endgroup::"
+ set +x
+ "$SCRIPT_DIR/pulsar_ci_tool.sh" move_test_reports
+ )
+}
+
# solution for printing output in "set -x" trace mode without tracing the echo calls
shopt -s expand_aliases
echo_and_restore_trace() {
@@ -30,35 +57,29 @@ echo_and_restore_trace() {
}
alias echo='{ [[ $- =~ .*x.* ]] && trace_enabled=1 || trace_enabled=0; set +x; } 2> /dev/null; echo_and_restore_trace'
-MVN_COMMAND='mvn -B -ntp'
-MVN_COMMAND_WITH_RETRY="build/retry.sh ${MVN_COMMAND}"
-MVN_TEST_COMMAND="${MVN_COMMAND_WITH_RETRY} test"
-
-echo -n "Test Group : $TEST_GROUP"
-
# Test Groups -- start --
-function broker_group_1() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='broker'
+function test_group_broker_group_1() {
+ mvn_test -pl pulsar-broker -Dgroups='broker'
}
-function broker_group_2() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other'
+function test_group_broker_group_2() {
+ mvn_test -pl pulsar-broker -Dgroups='schema,utils,functions-worker,broker-io,broker-discovery,broker-compaction,broker-naming,websocket,other'
}
-function broker_group_3() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='broker-admin'
+function test_group_broker_group_3() {
+ mvn_test -pl pulsar-broker -Dgroups='broker-admin'
}
-function broker_client_api() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='broker-api'
+function test_group_broker_client_api() {
+ mvn_test -pl pulsar-broker -Dgroups='broker-api'
}
-function broker_client_impl() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='broker-impl'
+function test_group_broker_client_impl() {
+ mvn_test -pl pulsar-broker -Dgroups='broker-impl'
}
-function broker_jdk8() {
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='broker-jdk8' -Dpulsar.allocator.pooled=true
+function test_group_broker_jdk8() {
+ mvn_test -pl pulsar-broker -Dgroups='broker-jdk8' -Dpulsar.allocator.pooled=true
}
# prints summaries of failed tests to console
@@ -81,7 +102,7 @@ function print_testng_failures() {
fi
local test_report_file="${testng_report_dir}/${failed_test_class}.txt"
if [ -f "${test_report_file}" ]; then
- local test_report="$(cat "${test_report_file}" | egrep "^Tests run: " | perl -p -se 's/^(Tests run: .*) <<< FAILURE! - in (.*)$/::warning::$report_prefix $2 - $1/' -- -report_prefix="${report_prefix}")"
+ local test_report="$(cat "${test_report_file}" | grep -E "^Tests run: " | perl -p -se 's/^(Tests run: .*) <<< FAILURE! - in (.*)$/::warning::$report_prefix $2 - $1/' -- -report_prefix="${report_prefix}")"
echo "$test_report"
cat "${test_report_file}"
fi
@@ -90,105 +111,80 @@ function print_testng_failures() {
)
}
-function broker_flaky() {
+function test_group_broker_flaky() {
echo "::endgroup::"
echo "::group::Running quarantined tests"
- $MVN_COMMAND test -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='' -DfailIfNoTests=false \
+ mvn_test --no-retry -pl pulsar-broker -Dgroups='quarantine' -DexcludedGroups='' -DfailIfNoTests=false \
-DtestForkCount=2 ||
print_testng_failures pulsar-broker/target/surefire-reports/testng-failed.xml "Quarantined test failure in" "Quarantined test failures"
echo "::endgroup::"
echo "::group::Running flaky tests"
- $MVN_TEST_COMMAND -pl pulsar-broker -Dgroups='flaky' -DtestForkCount=2
+ mvn_test -pl pulsar-broker -Dgroups='flaky' -DtestForkCount=2
echo "::endgroup::"
}
-function proxy() {
+function test_group_proxy() {
echo "::group::Running pulsar-proxy tests"
- $MVN_TEST_COMMAND -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceTlsStarterTest"
- $MVN_TEST_COMMAND -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceStarterTest"
- $MVN_TEST_COMMAND -pl pulsar-proxy -Dexclude='org.apache.pulsar.proxy.server.ProxyServiceTlsStarterTest,
+ mvn_test -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceTlsStarterTest"
+ mvn_test -pl pulsar-proxy -Dtest="org.apache.pulsar.proxy.server.ProxyServiceStarterTest"
+ mvn_test -pl pulsar-proxy -Dexclude='org.apache.pulsar.proxy.server.ProxyServiceTlsStarterTest,
org.apache.pulsar.proxy.server.ProxyServiceStarterTest'
echo "::endgroup::"
}
-function other() {
- $MVN_COMMAND_WITH_RETRY clean install -PbrokerSkipTest \
- -Dexclude='org/apache/pulsar/proxy/**/*.java,
- **/ManagedLedgerTest.java,
- **/TestPulsarKeyValueSchemaHandler.java,
- **/PrimitiveSchemaTest.java,
- BlobStoreManagedLedgerOffloaderTest.java'
+function test_group_other() {
+ mvn_test --install -PbrokerSkipTest \
+ -Dexclude='org/apache/pulsar/proxy/**/*.java,
+ **/ManagedLedgerTest.java,
+ **/TestPulsarKeyValueSchemaHandler.java,
+ **/PrimitiveSchemaTest.java,
+ BlobStoreManagedLedgerOffloaderTest.java'
- $MVN_TEST_COMMAND -pl managed-ledger -Dinclude='**/ManagedLedgerTest.java,
+ mvn_test -pl managed-ledger -Dinclude='**/ManagedLedgerTest.java,
**/OffloadersCacheTest.java'
- $MVN_TEST_COMMAND -pl pulsar-sql/presto-pulsar-plugin -Dinclude='**/TestPulsarKeyValueSchemaHandler.java'
+ mvn_test -pl pulsar-sql/presto-pulsar-plugin -Dinclude='**/TestPulsarKeyValueSchemaHandler.java'
- $MVN_TEST_COMMAND -pl pulsar-client -Dinclude='**/PrimitiveSchemaTest.java'
+ mvn_test -pl pulsar-client -Dinclude='**/PrimitiveSchemaTest.java'
- $MVN_TEST_COMMAND -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java'
+ mvn_test -pl tiered-storage/jcloud -Dinclude='**/BlobStoreManagedLedgerOffloaderTest.java'
echo "::endgroup::"
local modules_with_quarantined_tests=$(git grep -l '@Test.*"quarantine"' | grep '/src/test/java/' | \
- awk -F '/src/test/java/' '{ print $1 }' | egrep -v 'pulsar-broker|pulsar-proxy' | sort | uniq | \
+ awk -F '/src/test/java/' '{ print $1 }' | grep -v -E 'pulsar-broker|pulsar-proxy' | sort | uniq | \
perl -0777 -p -e 's/\n(\S)/,$1/g')
if [ -n "${modules_with_quarantined_tests}" ]; then
echo "::group::Running quarantined tests outside of pulsar-broker & pulsar-proxy (if any)"
- $MVN_COMMAND -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='' \
+ mvn_test --no-retry -pl "${modules_with_quarantined_tests}" test -Dgroups='quarantine' -DexcludedGroups='' \
-DfailIfNoTests=false || \
echo "::warning::There were test failures in the 'quarantine' test group."
echo "::endgroup::"
fi
}
+function list_test_groups() {
+ declare -F | awk '{print $NF}' | sort | grep -E '^test_group_' | sed 's/^test_group_//g' | tr '[:lower:]' '[:upper:]'
+}
+
# Test Groups -- end --
TEST_GROUP=$1
-
+if [ -z "$TEST_GROUP" ]; then
+ echo "usage: $0 [test_group]"
+ echo "Available test groups:"
+ list_test_groups
+ exit 1
+fi
+shift
+COMMANDLINE_ARGS=("$@")
echo "Test Group : $TEST_GROUP"
-
-set -x
-
-case $TEST_GROUP in
-
- BROKER_GROUP_1)
- broker_group_1
- ;;
-
- BROKER_GROUP_2)
- broker_group_2
- ;;
-
- BROKER_GROUP_3)
- broker_group_3
- ;;
-
- BROKER_CLIENT_API)
- broker_client_api
- ;;
-
- BROKER_CLIENT_IMPL)
- broker_client_impl
- ;;
-
- BROKER_FLAKY)
- broker_flaky
- ;;
-
- PROXY)
- proxy
- ;;
-
- OTHER)
- other
- ;;
-
- BROKER_JDK8)
- broker_jdk8
- ;;
-
- *)
- echo -n "INVALID TEST GROUP"
- exit 1
- ;;
-esac
+test_group_function_name="test_group_$(echo "$TEST_GROUP" | tr '[:upper:]' '[:lower:]')"
+if [[ "$(LC_ALL=C type -t "${test_group_function_name}")" == "function" ]]; then
+ set -x
+ eval "$test_group_function_name"
+else
+ echo "INVALID TEST GROUP"
+ echo "Available test groups:"
+ list_test_groups
+ exit 1
+fi
diff --git a/pulsar-io/docs/pom.xml b/pulsar-io/docs/pom.xml
index c9a593a2685f93..db2738725fcfcc 100644
--- a/pulsar-io/docs/pom.xml
+++ b/pulsar-io/docs/pom.xml
@@ -82,6 +82,16 @@
pulsar-io-debezium-postgres
${project.version}
+
+ ${project.groupId}
+ pulsar-io-debezium-oracle
+ ${project.version}
+
+
+ ${project.groupId}
+ pulsar-io-debezium-mssql
+ ${project.version}
+
${project.groupId}
pulsar-io-dynamodb
diff --git a/src/check-binary-license.sh b/src/check-binary-license.sh
index 5e8f365178acf4..bdf55b3a0b1696 100755
--- a/src/check-binary-license.sh
+++ b/src/check-binary-license.sh
@@ -97,32 +97,34 @@ done
if [ "$NO_PRESTO" -ne 1 ]; then
# check pulsar sql jars
JARS=$(tar -tf $TARBALL | grep '\.jar' | grep 'lib/presto/' | grep -v pulsar-client | grep -v bouncy-castle-bc | grep -v pulsar-metadata | grep -v 'managed-ledger' | grep -v 'pulsar-client-admin' | grep -v 'pulsar-client-api' | grep -v 'pulsar-functions-api' | grep -v 'pulsar-presto-connector-original' | grep -v 'pulsar-presto-distribution' | grep -v 'pulsar-common' | grep -v 'pulsar-functions-proto' | grep -v 'pulsar-functions-utils' | grep -v 'pulsar-io-core' | grep -v 'pulsar-transaction-common' | grep -v 'pulsar-package-core' | sed 's!.*/!!' | sort)
- LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
- LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
- LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp')
-
-
- for J in $JARS; do
- echo $J | grep -q "org.apache.pulsar"
- if [ $? == 0 ]; then
- continue
- fi
-
- echo "$LICENSE" | grep -q $J
- if [ $? != 0 ]; then
- echo $J unaccounted for in lib/presto/LICENSE
- EXIT=1
- fi
- done
-
- # Check all jars mentioned in LICENSE are bundled
- for J in $LICENSEJARS; do
- echo "$JARS" | grep -q $J
- if [ $? != 0 ]; then
- echo $J mentioned in lib/presto/LICENSE, but not bundled
- EXIT=2
- fi
- done
+ if [ -n "$JARS" ]; then
+ LICENSEPATH=$(tar -tf $TARBALL | awk '/^[^\/]*\/lib\/presto\/LICENSE/')
+ LICENSE=$(tar -O -xf $TARBALL "$LICENSEPATH")
+ LICENSEJARS=$(echo "$LICENSE" | sed -nE 's!.* (.*\.jar).*!\1!gp')
+
+
+ for J in $JARS; do
+ echo $J | grep -q "org.apache.pulsar"
+ if [ $? == 0 ]; then
+ continue
+ fi
+
+ echo "$LICENSE" | grep -q $J
+ if [ $? != 0 ]; then
+ echo $J unaccounted for in lib/presto/LICENSE
+ EXIT=1
+ fi
+ done
+
+ # Check all jars mentioned in LICENSE are bundled
+ for J in $LICENSEJARS; do
+ echo "$JARS" | grep -q $J
+ if [ $? != 0 ]; then
+ echo $J mentioned in lib/presto/LICENSE, but not bundled
+ EXIT=2
+ fi
+ done
+ fi
fi
if [ $EXIT != 0 ]; then