forked from opensearch-project/opensearch-build
-
Notifications
You must be signed in to change notification settings - Fork 0
/
setup_runners.sh
executable file
·272 lines (236 loc) · 13.3 KB
/
setup_runners.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
#!/bin/bash
# Copyright OpenSearch Contributors
# SPDX-License-Identifier: Apache-2.0
#
# The OpenSearch Contributors require contributions made to
# this file be licensed under the Apache-2.0 license or a
# compatible open source license.
###### Information ############################################################################
# Name: setup_runners.sh
#
# About: 1. Run instances on EC2 based on parameters defined and wait for completion
# 2. SSH to these instances and configure / bootstrap on $GIT_URL_REPO as runners
# 3. Unbootstrap the runners and terminate the instances for cleanups
#
# Usage: ./setup_runners.sh $ACTION $EC2_INSTANCE_NAMES $GITHUB_TOKEN
# $ACTION: run | terminate (required)
# $EC2_INSTANCE_NAMES: <names of instances> (required, sep ",")
# $GITHUB_TOKEN: GitHub PAT with repo scope and Admin Access to $GIT_URL_REPO
#
# Requirements: The env that runs this script must have its AWS resources with these configurations
#
# 1. Have an AWS user account with access to EC2 resource, remember the User ID
#
# 2. Create EC2 keypairs with name "odfe-release-runner"
#
# 3. Create EC2 Security Group with name "odfe-release-runner"
# with inbound rules of 22/9200/9600/5601 from IP ranges that need access to the runner
#
# 4. Create IAM resources:
#
# * IAM role with name "odfe-release-runner", and these policies attached to it:
# i. AmazonEC2RoleforSSM
# ii. AmazonSSMManagedInstanceCore
#
# * IAM user "opendistro-ec2-user", generate a pair of security credentials,
# and these policies attached to it:
# i. AmazonEC2FullAccess
# ii. Custom policy using this json, I name it again to "odfe-release-runner"
# {
# "Version": "2012-10-17",
# "Statement": [
# {
# "Sid": "VisualEditor0",
# "Effect": "Allow",
# "Action": [
# "ssm:SendCommand",
# "iam:PassRole"
# ],
# "Resource": [
# "arn:aws:ssm:*:*:document/*",
# "arn:aws:ec2:*:*:instance/*",
# "arn:aws:iam::<AWS User ID>:role/<IAM Role Name above>"
# ]
# },
# {
# "Sid": "VisualEditor1",
# "Effect": "Allow",
# "Action": "ssm:DescribeInstanceInformation",
# "Resource": "*"
# }
# ]
# }
#
# 5. awscli must "aws login" with the security credencial created for IAM user
# in the step 4 above
#
# 6. If you change the above resources name from "odfe-release-runner" to "xyz",
# please update "Variables / Parameters / Settings" section of this script
#
# 7. Runner AMI requires installation of packages of these (java version can be different as gradle might request a higher version):
# Debian:
# sudo apt install -y curl wget unzip tar jq python python3 git awscli openjdk-14-jdk
# sudo apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
#
# RedHat:
# sudo yum install -y curl wget unzip tar jq python python3 git awscli java-latest-openjdk
# sudo yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib
#
# Also you need to install java devel if you want to compile library (e.g. knnlib)
#
# 8. AMI must be at least 16GB during the creation.
#
# 9. You can use `export GIT_UTL_REPO="opendistro-for-elasticsearch/opendistro-build"` or similar to set the Git Repo of the runner
#
# 10. JDK & SSM Agent
# You should find a way to install JDK14 or later on the server
# Debian with: sudo add-apt-repository ppa:openjdk-r/ppa
# RedHat with: https://fedoraproject.org/wiki/EPEL
#
# Also, you need to install ssm agent
# on non-al2 machine due to ssm RunCommand requires that
# https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-manual-agent-install.html
#
# us-west-2
# RPM x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_amd64/amazon-ssm-agent.rpm
# RPM arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_arm64/amazon-ssm-agent.rpm
# DEB x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_amd64/amazon-ssm-agent.deb
# DEB arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_arm64/amazon-ssm-agent.deb
# yum or dpkg then systemctl enable/start amazon-ssm-agent
#
# 11. You also need to set the user of the GitHub Token to have ADMIN access of the GitHub Repo
# So that runner can be successfully bootstrapped to action tab in settings.
#
###############################################################################################
set -e
#####################################
# Variables / Parameters / Settings #
#####################################
# This script allows users to manually assign parameters
if [ "$#" -lt 3 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]
then
echo "Please assign at least 3 parameters when running this script"
echo "Example: $0 \$ACTION \$EC2_INSTANCE_NAMES(,) \$GITHUB_TOKEN, \$EC2_AMI_ID"
echo "Example (run must have 4 parameters): $0 \"run\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"<GitHub PAT>\" \"ami-*\""
echo "Example (terminate must have 3 parameters): $0 \"terminate\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"<GitHub PAT>\""
echo "You can use \`export GIT_UTL_REPO=\"opendistro-for-elasticsearch/opendistro-build\"\` or similar to set the Git Repo of the runner"
exit 1
fi
SETUP_ACTION=$1
SETUP_RUNNER=`echo $2 | sed 's/,/ /g'`
SETUP_GIT_TOKEN=$3
# AMI on us-west-2
# Distro Arch Recommand Username AMI-ID Java Comments
# RPM-al2 x64 YES ec2-user ami-0bd968fea932935f4 none no jdk + reports kibana dependencies
# RPM-al2 arm64 YES ec2-user ami-0ef0c96643bbd01f2 jdk14 preinstall with tar.gz + reports kibana dependencies
# DEB-ubu1804 arm64 YES ubuntu ami-03f8a33a16290a84c jdk14 preinstall + docker + docker compose + reports kibana dependencies
# RPM-centos8 x64 NO centos ami-011f59f50bac33376 jdk15 preinstall
# RPM-centos8 arm64 NO centos ami-0ed17173ab64255b1 jdk15 preinstall
EC2_AMI_ID=$4
if [ "$SETUP_ACTION" = "run" ]
then
if [ -z "$EC2_AMI_ID" ]
then
echo " \$EC2_AMI_ID is empty, please add a 4th parameter for the run "
exit 1
else
# This does not support MacOS now due to cumbersome descriptions
# MacOS sample: ami-00b3e436dc75183e0
# "PlatformDetails": "Linux/UNIX"
# "Architecture": "x86_64_mac"
EC2_AMI_PLATFORM=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].PlatformDetails' --output text | awk -F '/' '{print $1}' | tr '[:upper:]' '[:lower:]'`
EC2_AMI_ARCH=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Architecture' --output text | sed 's/x86_64/x64/g'`
EC2_AMI_NAME=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Name' --output text | tr '[:upper:]' '[:lower:]'`
EC2_AMI_USER="ec2-user"; if echo $EC2_AMI_NAME | grep "centos"; then EC2_AMI_USER="centos"; elif echo $EC2_AMI_NAME | grep "ubuntu"; then EC2_AMI_USER="ubuntu"; fi
EC2_INSTANCE_TYPE="m5.xlarge"; if [ "$EC2_AMI_ARCH" = "arm64" ]; then EC2_INSTANCE_TYPE="m6g.xlarge"; fi
RUNNER_URL=`curl -s https://api.github.com/repos/actions/runner/releases/latest -H "Authorization: token $SETUP_GIT_TOKEN" | jq -r '.assets[].browser_download_url' | grep "$EC2_AMI_PLATFORM" | grep "$EC2_AMI_ARCH" | tail -n 1`
echo Provision $EC2_AMI_PLATFORM $EC2_AMI_ARCH $EC2_AMI_NAME $EC2_AMI_USER $EC2_INSTANCE_TYPE $RUNNER_URL
fi
fi
EC2_INSTANCE_SIZE=20 #GiB
EC2_KEYPAIR="odfe-release-runner"
EC2_SECURITYGROUP="odfe-release-runner"
IAM_ROLE="odfe-release-runner"
GIT_URL_API="https://api.github.com/repos"
GIT_URL_BASE="https://github.com"
GIT_URL_REPO=${GIT_URL_REPO:-opensearch-project/opensearch-build}
RUNNER_DIR="actions-runner"
echo "###############################################"
echo "Start Running $0 $1 $2"
echo "###############################################"
###############################################
# Run / Start instances and bootstrap runners #
###############################################
if [ "$SETUP_ACTION" = "run" ]
then
echo "GIT_URL_REPO $GIT_URL_REPO"
echo ""
echo "Run / Start instances and bootstrap runners [${SETUP_RUNNER}]"
echo ""
# Get information
instance_root_device=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].RootDeviceName' --output text`
# Provision VMs
for instance_name1 in $SETUP_RUNNER
do
echo "[${instance_name1}]: Start provisioning vm"
aws ec2 run-instances --image-id $EC2_AMI_ID --count 1 --instance-type $EC2_INSTANCE_TYPE \
--block-device-mapping DeviceName=$instance_root_device,Ebs={VolumeSize=$EC2_INSTANCE_SIZE} \
--key-name $EC2_KEYPAIR --security-groups $EC2_SECURITYGROUP \
--iam-instance-profile Name=$IAM_ROLE \
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$instance_name1}]" > /dev/null 2>&1; echo $?
sleep 1
done
echo ""
echo "Sleep for 120 seconds for EC2 instances to start running"
echo ""
sleep 120
# Setup VMs to register as runners
for instance_name2 in $SETUP_RUNNER
do
echo "[${instance_name2}]: Make change of the runner hostname"
aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \
--parameters '{"commands": ["#!/bin/bash", "sudo hostnamectl set-hostname '${instance_name2}'"]}' \
--output text > /dev/null 2>&1; echo $?
echo "[${instance_name2}]: Get latest runner binary to server ${RUNNER_URL}"
aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \
--parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"mkdir -p '${RUNNER_DIR}' && cd '${RUNNER_DIR}' && wget -q '${RUNNER_URL}' && tar -xzf *.tar.gz && rm *.tar.gz \""]}' \
--output text > /dev/null 2>&1; echo $?
echo "[${instance_name2}]: Get runner token and bootstrap on Git"
instance_runner_token=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request POST "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/registration-token" | jq -r .token`
# Wait 10 seconds for untar of runner binary to complete
aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \
--parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"sleep 30 && cd '${RUNNER_DIR}' && ./config.sh --unattended --url '${GIT_URL_BASE}/${GIT_URL_REPO}' --labels '${instance_name2}' --token '${instance_runner_token}' && nohup ./run.sh &\""]}' \
--output text > /dev/null 2>&1; echo $?
sleep 5
done
echo ""
echo "Wait for 90 seconds for runners to bootstrap on Git"
echo ""
sleep 90
echo ""
echo "All runners are online on Git"
echo ""
fi
###################################################
# Terminate / Delete instances and remove runners #
###################################################
if [ "$SETUP_ACTION" = "terminate" ]
then
echo "GIT_URL_REPO $GIT_URL_REPO"
echo ""
echo "Terminate / Delete instances and remove runners [${SETUP_RUNNER}]"
echo ""
for instance_name3 in $SETUP_RUNNER
do
instance_runner_id_git=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request GET "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners" | jq ".runners[] | select(.name == \"${instance_name3}\") | .id"`
echo "[${instance_name3}]: Unbootstrap runner from Git"
curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request DELETE "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/${instance_runner_id_git}"; echo $?
instance_runner_id_ec2=`aws ec2 describe-instances --filters "Name=tag:Name,Values=$instance_name3" | jq -r '.Reservations[].Instances[] | select(.State.Code == 16) | .InstanceId'` # Only running instances
echo "[${instance_name3}]: Remove tags Name"
aws ec2 delete-tags --resources $instance_runner_id_ec2 --tags Key=Name > /dev/null 2>&1; echo $?
echo "[${instance_name3}]: Terminate runner"
aws ec2 terminate-instances --instance-ids $instance_runner_id_ec2 > /dev/null 2>&1; echo $?
sleep 1
done
echo "All runners are offline on Git"
fi