Skip to content

Commit 589a84b

Browse files
mwang2016mwang2016
committed
Add github actions ci pipeline
1 parent 92ef8ba commit 589a84b

File tree

1 file changed

+187
-0
lines changed

1 file changed

+187
-0
lines changed

‎.github/workflows/ci_build_test.yaml

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
name: CI Build Test
2+
3+
on:
4+
pull_request:
5+
branches-ignore:
6+
- gh-pages
7+
8+
jobs:
9+
10+
build-unit-test:
11+
name: build and run unit test
12+
runs-on: ubuntu-20.04
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v2
16+
17+
- name: Get maven dependencies
18+
run: |
19+
mvn dependency:go-offline
20+
21+
- name: Builder
22+
run: |
23+
mvn compile
24+
25+
- name: Cache pkg
26+
uses: actions/cache@v1
27+
with:
28+
path: ~/.m2
29+
key: kafka-connect-splunk-${{ matrix.kafka_version }}
30+
31+
- name: Run Unit tests
32+
run: |
33+
mvn package -Dsurefire.useSystemClassLoader=false -q
34+
35+
- name: Publish Unit Test Results
36+
uses: EnricoMi/publish-unit-test-result-action/composite@v1
37+
if: always()
38+
with:
39+
check_name: Unit Test Results
40+
files: "target/surefire-reports/*.xml"
41+
42+
e2e_test:
43+
name: e2e test - kafka version-${{ matrix.kafka_version }}
44+
runs-on: ubuntu-20.04
45+
needs:
46+
- build-unit-test
47+
strategy:
48+
matrix:
49+
kafka_version:
50+
- "1.1.1"
51+
- "2.0.0"
52+
- "2.1.0"
53+
- "2.6.0"
54+
kafka_package:
55+
- "kafka_2.11-1.1.1.tgz"
56+
- "kafka_2.11-2.0.0.tgz"
57+
- "kafka_2.12-2.1.0.tgz"
58+
- "kafka_2.13-2.6.0.tgz"
59+
env:
60+
MINIKUBE_HOME: /home/circleci
61+
CI_SPLUNK_VERSION: "8.0.2.1"
62+
CI_SPLUNK_FILENAME: splunk-8.0.2.1-f002026bad55-linux-2.6-amd64.deb
63+
CI_SPLUNK_HOST: 127.0.0.1
64+
CI_SPLUNK_PORT: 8089
65+
CI_SPLUNK_USERNAME: admin
66+
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
67+
CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
68+
CI_SPLUNK_PASSWORD: helloworld
69+
CI_INDEX_EVENTS: main
70+
CI_KAFKA_HEADER_INDEX: kafka
71+
CI_DATAGEN_IMAGE: rock1017/log-generator:latest
72+
CI_OLD_CONNECTOR_VERSION: v1.2.0
73+
74+
steps:
75+
- name: Checkout
76+
uses: actions/checkout@v2
77+
78+
- uses: actions/cache@v2
79+
with:
80+
path: ~/.m2
81+
key: kafka-connect-splunk-${{ matrix.kafka_version }}
82+
83+
- name: Install Splunk
84+
run: |
85+
cd /tmp && wget -O $CI_SPLUNK_FILENAME 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version='$CI_SPLUNK_VERSION'&product=splunk&filename='$CI_SPLUNK_FILENAME'&wget=true'
86+
sudo dpkg -i $CI_SPLUNK_FILENAME
87+
# Set user seed
88+
hashed_pwd=$(sudo /opt/splunk/bin/splunk hash-passwd $CI_SPLUNK_PASSWORD)
89+
sudo tee /opt/splunk/etc/system/local/user-seed.conf > /dev/null \<< EOF
90+
[user_info]
91+
USERNAME = $CI_SPLUNK_USERNAME
92+
HASHED_PASSWORD = $hashed_pwd
93+
EOF
94+
# Add delete capability to admin role
95+
sudo tee -a /opt/splunk/etc/system/local/authorize.conf > /dev/null \<< EOF
96+
[role_admin]
97+
delete_by_keyword = enabled
98+
EOF
99+
100+
# Start Splunk
101+
sudo /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt
102+
# Enable HEC services
103+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
104+
# Setup HEC token
105+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=$CI_SPLUNK_HEC_TOKEN" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
106+
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
107+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token_ack&token=$CI_SPLUNK_HEC_TOKEN_ACK&useACK=1" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
108+
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token_ack -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
109+
110+
# Setup Indexes
111+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_INDEX_EVENTS&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
112+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_KAFKA_HEADER_INDEX&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
113+
sudo /opt/splunk/bin/splunk restart --accept-license --answer-yes --no-prompt
114+
115+
- name: Install Kafka ${{ matrix.kafka_version }}
116+
run: |
117+
cd /tmp && wget https://archive.apache.org/dist/kafka/<< parameters.kafka_version >>/<< parameters.kafka_package >>
118+
sudo tar xzf << parameters.kafka_package >>
119+
rm << parameters.kafka_package >>
120+
sudo mv kafka_* /usr/local/kafka
121+
sudo tee /etc/systemd/system/zookeeper.service > /dev/null \<< EOF
122+
[Unit]
123+
Description=Apache Zookeeper server
124+
Documentation=http://zookeeper.apache.org
125+
Requires=network.target remote-fs.target
126+
After=network.target remote-fs.target
127+
[Service]
128+
Type=simple
129+
ExecStart=/usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties
130+
ExecStop=/usr/local/kafka/bin/zookeeper-server-stop.sh
131+
Restart=on-abnormal
132+
[Install]
133+
WantedBy=multi-user.target
134+
EOF
135+
136+
sudo tee /etc/systemd/system/kafka.service > /dev/null \<< EOF
137+
[Unit]
138+
Description=Apache Kafka Server
139+
Documentation=http://kafka.apache.org/documentation.html
140+
Requires=zookeeper.service
141+
[Service]
142+
Type=simple
143+
Environment="JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
144+
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
145+
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
146+
[Install]
147+
WantedBy=multi-user.target
148+
EOF
149+
150+
- name: Start Kafka Server
151+
run: |
152+
sudo systemctl daemon-reload
153+
sudo systemctl start zookeeper
154+
sudo systemctl start kafka
155+
156+
- uses: actions/setup-python@v2
157+
with:
158+
python-version: 3.7
159+
160+
- name: Test kafka connect upgrade
161+
run: |
162+
echo "Download kafka connect "$CI_OLD_CONNECTOR_VERSION
163+
sudo mkdir -p /usr/local/share/kafka/plugins/
164+
wget https://github.com/splunk/kafka-connect-splunk/releases/download/$CI_OLD_CONNECTOR_VERSION/splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar
165+
sudo cp splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar /usr/local/share/kafka/plugins/
166+
sed -i 's/plugin\.path\=connectors\//plugin\.path\=\/usr\/local\/share\/kafka\/plugins\//' /home/circleci/repo/config/connect-distributed-quickstart.properties
167+
sed -i 's/key\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/key\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
168+
sed -i 's/value\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/value\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
169+
pyenv global 3.6.5
170+
pip install --upgrade pip
171+
pip install -r test/requirements.txt
172+
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
173+
echo "Test kafka connect upgrade ..."
174+
python test/lib/connector_upgrade.py
175+
176+
- name: Install kafka connect
177+
run: |
178+
sudo rm -f /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar
179+
sudo cp /home/circleci/repo/target/splunk-kafka-connect*.jar /usr/local/share/kafka/plugins/
180+
test -f /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar && echo /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar
181+
sudo /usr/local/kafka/bin/connect-distributed.sh /home/circleci/repo/config/connect-distributed-quickstart.properties &
182+
183+
- name: Run functional tests
184+
run: |
185+
sleep 5
186+
echo "Running functional tests....."
187+
python -m pytest -p no:warnings -s

0 commit comments

Comments
 (0)