-
Notifications
You must be signed in to change notification settings - Fork 3.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[CI] Combine multiple GitHub Actions workflows into a smaller amount …
…of workflows
- Loading branch information
Showing
47 changed files
with
1,379 additions
and
2,860 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
|
|
||
| name: clean disk | ||
| description: makes some more space available on the disk by removing files | ||
| inputs: | ||
| mode: | ||
| description: "Use 'full' to clean as much as possible" | ||
| required: false | ||
| runs: | ||
| using: composite | ||
| steps: | ||
| - run: | | ||
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | ||
| directories=(/usr/local/lib/android /opt/ghc) | ||
| if [[ "${{ inputs.mode }}" == "full" ]]; then | ||
| # remove these directories only when mode is 'full' | ||
| directories+=(/usr/share/dotnet) | ||
| fi | ||
| emptydir=/tmp/empty$$/ | ||
| mkdir $emptydir | ||
| echo "::group::Available diskspace" | ||
| time df -BM / /mnt | ||
| echo "::endgroup::" | ||
| for directory in "${directories[@]}"; do | ||
| echo "::group::Removing $directory" | ||
| # fast way to delete a lot of files on linux | ||
| time sudo eatmydata rsync -a --delete $emptydir ${directory}/ | ||
| time sudo eatmydata rm -rf ${directory} | ||
| time df -BM / /mnt | ||
| echo "::endgroup::" | ||
| done | ||
| echo "::group::Cleaning apt state" | ||
| time sudo bash -c "apt-get clean; apt-get autoclean; apt-get -y --purge autoremove" | ||
| time df -BM / /mnt | ||
| echo "::endgroup::" | ||
| fi | ||
| echo "::group::Available diskspace" | ||
| time df -BM / /mnt | ||
| echo "::endgroup::" | ||
| shell: bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
|
|
||
| name: Copy test reports | ||
| description: Aggregates all test reports to ./test-reports and ./surefire-reports directories | ||
| runs: | ||
| using: composite | ||
| steps: | ||
| - run: | | ||
| $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh move_test_reports | ||
| shell: bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
|
|
||
| name: Merge test reports to a single file | ||
| description: Merges junit test reports to a single file | ||
| inputs: | ||
| summary_title: | ||
| description: 'The title for the summary' | ||
| required: false | ||
| runs: | ||
| using: composite | ||
| steps: | ||
| - run: | | ||
| if ls test-reports/*.xml 2>/dev/null; then | ||
| echo '::group::Installing tooling...' | ||
| # this requires "runs-on: ubuntu-20.04" since junitparser package is not available in previous Ubuntu version | ||
| sudo apt-get -y install xmlstarlet junitparser >/dev/null | ||
| echo '::endgroup::' | ||
| echo '::group::Parsing and merging test reports...' | ||
| junitparser merge test-reports/*.xml merged-test-report.xml || true | ||
| echo '::endgroup::' | ||
| if [[ -f merged-test-report.xml && -n "${{ inputs.summary_title }}" ]]; then | ||
| xmlstarlet sel -T -t -m /testsuites -o "::warning::${{ inputs.summary_title }} " -v '@tests' -o ' tests, ' -v '@failures' -o ' failures, ' -v '@errors' -o ' errors, ' -v '@skipped' -o ' skipped, duration ' -v 'number(@time)' -o 's' merged-test-report.xml | ||
| fi | ||
| else | ||
| echo 'No test-reports/*.xml files found.' | ||
| fi | ||
| shell: bash |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| # | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
| # | ||
|
|
||
| name: ssh access | ||
| description: Sets up SSH access to build VM with upterm | ||
| inputs: | ||
| action: | ||
| description: | | ||
| Action to perform: options are "start" and "wait" | ||
| "start" will install, configure and start upterm. | ||
| "wait" will wait until a connection is established to upterm and will continue to wait until the session is closed. | ||
| required: false | ||
| default: 'start' | ||
| limit-access-to-actor: | ||
| description: 'If only the public SSH keys of the user triggering the workflow should be authorized' | ||
| required: false | ||
| default: 'false' | ||
| limit-access-to-users: | ||
| description: 'If only the public SSH keys of the listed GitHub users should be authorized. Comma separate list of GitHub user names.' | ||
| required: false | ||
| default: '' | ||
| secure-access: | ||
| description: | | ||
| Set to false for allowing public access when limit-access-to-actor and limit-access-to-users are unset. | ||
| required: false | ||
| default: 'true' | ||
| timeout: | ||
| description: 'When action=wait, the timeout in seconds to wait for the user to connect' | ||
| required: false | ||
| default: '300' | ||
| runs: | ||
| using: composite | ||
| steps: | ||
| - run: | | ||
| if [[ "${{ inputs.action }}" == "start" ]]; then | ||
| echo "::group::Installing upterm & tmux" | ||
| if [[ "$OSTYPE" == "linux-gnu"* ]]; then | ||
| # install upterm | ||
| curl -sL https://github.com/owenthereal/upterm/releases/download/v0.7.6/upterm_linux_amd64.tar.gz | tar zxvf - -C /tmp upterm && sudo install /tmp/upterm /usr/local/bin/ && rm -rf /tmp/upterm | ||
| # install tmux if it's not present | ||
| if ! command -v tmux &>/dev/null; then | ||
| sudo apt-get -y install tmux | ||
| fi | ||
| elif [[ "$OSTYPE" == "darwin"* ]]; then | ||
| brew install owenthereal/upterm/upterm | ||
| # install tmux if it's not present | ||
| if ! command -v tmux &>/dev/null; then | ||
| brew install tmux | ||
| fi | ||
| else | ||
| echo "Unsupported $OSTYPE" | ||
| exit 0 | ||
| fi | ||
| echo '::endgroup::' | ||
| echo "::group::Configuring ssh and ssh keys" | ||
| # generate ssh key | ||
| mkdir -p ~/.ssh | ||
| chmod 0700 ~/.ssh | ||
| if [ ! -f ~/.ssh/id_rsa ]; then | ||
| ssh-keygen -q -t rsa -N "" -f ~/.ssh/id_rsa | ||
| fi | ||
| if [ ! -f ~/.ssh/id_ed25519 ]; then | ||
| ssh-keygen -q -t ed25519 -N "" -f ~/.ssh/id_ed25519 | ||
| fi | ||
| # configure ssh | ||
| echo -e "Host *\nStrictHostKeyChecking no\nCheckHostIP no\nTCPKeepAlive yes\nServerAliveInterval 30\nServerAliveCountMax 180\nVerifyHostKeyDNS yes\nUpdateHostKeys yes\n" > ~/.ssh/config | ||
| # Auto-generate ~/.ssh/known_hosts by attempting connection to uptermd.upterm.dev | ||
| ssh -i ~/.ssh/id_ed25519 uptermd.upterm.dev || true | ||
| # @cert-authority entry is a mandatory entry when connecting to upterm. generate the entry based on the known_hosts entry key | ||
| cat <(cat ~/.ssh/known_hosts | awk '{ print "@cert-authority * " $2 " " $3 }') >> ~/.ssh/known_hosts | ||
| authorizedKeysParameter="" | ||
| authorizedKeysFile=${HOME}/.ssh/authorized_keys | ||
| if [[ "${{ inputs.secure-access }}" != "false" ]]; then | ||
| ssh-keygen -q -t ed25519 -N "$(echo $RANDOM | md5sum | awk '{ print $1 }')" -C "Prevent public access" -f /tmp/dummykey$$ | ||
| cat /tmp/dummykey$$.pub >> $authorizedKeysFile | ||
| rm /tmp/dummykey$$ /tmp/dummykey$$.pub | ||
| fi | ||
| limit_access_to_actor="${{ inputs.limit-access-to-actor }}" | ||
| if [[ "${limit_access_to_actor}" == "true" ]]; then | ||
| echo "Adding ${GITHUB_ACTOR} to allowed users (identified by ssh key registered in GitHub)" | ||
| curl -s https://github.com/${GITHUB_ACTOR}.keys >> $authorizedKeysFile | ||
| fi | ||
| limit_access_to_users="${{ inputs.limit-access-to-users }}" | ||
| for github_user in ${limit_access_to_users//,/ }; do | ||
| if [[ -n "${github_user}" ]]; then | ||
| echo "Adding ${github_user} to allowed users (identified by ssh key registered in GitHub)" | ||
| curl -s https://github.com/${github_user}.keys >> $authorizedKeysFile | ||
| fi | ||
| done | ||
| if [ -f $authorizedKeysFile ]; then | ||
| chmod 0600 $authorizedKeysFile | ||
| authorizedKeysParameter="-a $authorizedKeysFile" | ||
| echo -e "Using $authorizedKeysFile\nContent:\n---------------------------" | ||
| cat $authorizedKeysFile | ||
| echo "---------------------------" | ||
| fi | ||
| echo '::endgroup::' | ||
| echo "::group::Starting terminal session and connecting to server" | ||
| tmux new -d -s upterm-wrapper -x 132 -y 43 "upterm host ${authorizedKeysParameter} --force-command 'tmux attach -t upterm' -- tmux new -s upterm -x 132 -y 43" | ||
| sleep 2 | ||
| tmux send-keys -t upterm-wrapper q C-m | ||
| sleep 1 | ||
| tmux set -t upterm-wrapper window-size largest | ||
| tmux set -t upterm window-size largest | ||
| echo '::endgroup::' | ||
| echo -e "\nSSH connection information" | ||
| shopt -s nullglob | ||
| upterm session current --admin-socket ~/.upterm/*.sock | ||
| elif [[ "${{ inputs.action }}" == "wait" ]]; then | ||
| # only wait if upterm was installed | ||
| if command -v upterm &>/dev/null; then | ||
| shopt -s nullglob | ||
| echo "SSH connection information" | ||
| upterm session current --admin-socket ~/.upterm/*.sock || { | ||
| echo "upterm isn't running. Not waiting any longer." | ||
| exit 0 | ||
| } | ||
| timeout=${{ inputs.timeout }} | ||
| echo "Waiting $timeout seconds..." | ||
| sleep $timeout | ||
| echo "Keep waiting as long as there's a connected session" | ||
| while upterm session current --admin-socket ~/.upterm/*.sock|grep Connected &>/dev/null; do | ||
| sleep 30 | ||
| done | ||
| echo "No session is connected. Not waiting any longer." | ||
| else | ||
| echo "upterm isn't installed" | ||
| fi | ||
| fi | ||
| shell: bash |
Oops, something went wrong.