Skip to content

Commit 87a7f99

Browse files
mwang2016mwang2016
committed
Add github actions ci pipeline
1 parent 92ef8ba commit 87a7f99

File tree

2 files changed

+222
-0
lines changed

2 files changed

+222
-0
lines changed

‎.github/workflows/ci_build_test.yaml

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
name: CI Build Test
2+
3+
on:
4+
pull_request:
5+
branches-ignore:
6+
- gh-pages
7+
8+
jobs:
9+
10+
build-unit-test:
11+
name: build and run unit test
12+
runs-on: ubuntu-20.04
13+
steps:
14+
- name: Checkout
15+
uses: actions/checkout@v2
16+
17+
- name: Get maven dependencies
18+
run: |
19+
mvn dependency:go-offline
20+
21+
- name: Builder
22+
run: |
23+
mvn compile
24+
25+
- name: Cache pkg
26+
uses: actions/cache@v1
27+
with:
28+
path: ~/.m2
29+
key: kafka-connect-splunk-${{ matrix.kafka_version }}
30+
31+
- name: Run Unit tests
32+
run: |
33+
mvn package -Dsurefire.useSystemClassLoader=false -q
34+
35+
- name: Publish Unit Test Results
36+
uses: EnricoMi/publish-unit-test-result-action/composite@v1
37+
if: always()
38+
with:
39+
check_name: Unit Test Results
40+
files: "target/surefire-reports/*.xml"
41+
42+
e2e_test:
43+
name: e2e test - kafka version-${{ matrix.kafka_version }}
44+
runs-on: ubuntu-20.04
45+
needs:
46+
- build-unit-test
47+
strategy:
48+
matrix:
49+
include:
50+
- kafka_version: "1.1.1"
51+
kafka_package: "kafka_2.11-1.1.1.tgz"
52+
- kafka_version: "2.0.0"
53+
kafka_package: "kafka_2.11-2.0.0.tgz"
54+
- kafka_version: "2.1.0"
55+
kafka_package: "kafka_2.12-2.1.0.tgz"
56+
- kafka_version: "2.6.0"
57+
kafka_package: "kafka_2.13-2.6.0.tgz"
58+
env:
59+
CI_SPLUNK_VERSION: "8.2.0"
60+
CI_SPLUNK_FILENAME: splunk-8.2.0-e053ef3c985f-Linux-x86_64.tgz
61+
CI_SPLUNK_HOST: 127.0.0.1
62+
CI_SPLUNK_PORT: 8089
63+
CI_SPLUNK_USERNAME: admin
64+
CI_SPLUNK_HEC_TOKEN: a6b5e77f-d5f6-415a-bd43-930cecb12959
65+
CI_SPLUNK_HEC_TOKEN_ACK: a6b5e77f-d5f6-415a-bd43-930cecb12950
66+
CI_SPLUNK_PASSWORD: helloworld
67+
CI_INDEX_EVENTS: main
68+
CI_KAFKA_HEADER_INDEX: kafka
69+
CI_DATAGEN_IMAGE: rock1017/log-generator:latest
70+
CI_OLD_CONNECTOR_VERSION: v1.2.0
71+
KUBERNETES_VERSION: v1.15.2
72+
MINIKUBE_VERSION: v1.21.0
73+
74+
steps:
75+
- name: Checkout
76+
uses: actions/checkout@v2
77+
78+
- name: Install Splunk
79+
run: |
80+
cd /opt && wget -O $CI_SPLUNK_FILENAME 'https://www.splunk.com/bin/splunk/DownloadActivityServlet?architecture=x86_64&platform=linux&version='$CI_SPLUNK_VERSION'&product=splunk&filename='$CI_SPLUNK_FILENAME'&wget=true'
81+
sudo tar xzvf $CI_SPLUNK_FILENAME
82+
83+
# Set user seed
84+
hashed_pwd=$(sudo /opt/splunk/bin/splunk hash-passwd $CI_SPLUNK_PASSWORD)
85+
# sudo tee /opt/splunk/etc/system/local/user-seed.conf > /dev/null \<< EOF
86+
sudo tee /opt/splunk/etc/system/local/user-seed.conf <<EOF >/dev/null
87+
[user_info]
88+
USERNAME = $CI_SPLUNK_USERNAME
89+
HASHED_PASSWORD = $hashed_pwd
90+
EOF
91+
# Add delete capability to admin role
92+
sudo tee /opt/splunk/etc/system/local/authorize.conf <<EOF >/dev/null
93+
[role_admin]
94+
delete_by_keyword = enabled
95+
EOF
96+
echo "... ... ..."
97+
sudo /opt/splunk/bin/splunk start --accept-license --answer-yes --no-prompt
98+
# Enable HEC services
99+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http/http/enable
100+
# Create new HEC token
101+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token&token=$CI_SPLUNK_HEC_TOKEN" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
102+
# Enable HEC new-token
103+
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
104+
# Create new HEC token with ack
105+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=splunk_hec_token_ack&token=$CI_SPLUNK_HEC_TOKEN_ACK&useACK=1" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/nobody/splunk_httpinput/data/inputs/http
106+
# Enable HEC new-token
107+
sudo /opt/splunk/bin/splunk http-event-collector enable -name splunk_hec_token_ack -uri https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT -auth $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD
108+
# Setup Indexes
109+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_INDEX_EVENTS&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
110+
curl -X POST -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD -k -d "name=$CI_KAFKA_HEADER_INDEX&datatype=event" https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/servicesNS/-/search/data/indexes
111+
# Restart Splunk
112+
curl -k -u $CI_SPLUNK_USERNAME:$CI_SPLUNK_PASSWORD https://$CI_SPLUNK_HOST:$CI_SPLUNK_PORT/services/server/control/restart -X POST
113+
114+
- uses: actions/cache@v2
115+
with:
116+
path: ~/.m2
117+
key: kafka-connect-splunk-${{ matrix.kafka_version }}
118+
119+
- name: Install Kafka ${{ matrix.kafka_version }}
120+
run: |
121+
cd /tmp && wget https://archive.apache.org/dist/kafka/${{ matrix.kafka_version }}/${{ matrix.kafka_package }}
122+
sudo tar xzf ${{ matrix.kafka_package }}
123+
rm ${{ matrix.kafka_package }}
124+
sudo mv kafka_* /usr/local/kafka
125+
sudo tee /etc/systemd/system/zookeeper.service <<EOF >/dev/null
126+
[Unit]
127+
Description=Apache Zookeeper server
128+
Documentation=http://zookeeper.apache.org
129+
Requires=network.target remote-fs.target
130+
After=network.target remote-fs.target
131+
[Service]
132+
Type=simple
133+
ExecStart=/usr/local/kafka/bin/zookeeper-server-start.sh /usr/local/kafka/config/zookeeper.properties
134+
ExecStop=/usr/local/kafka/bin/zookeeper-server-stop.sh
135+
Restart=on-abnormal
136+
[Install]
137+
WantedBy=multi-user.target
138+
EOF
139+
140+
sudo tee /etc/systemd/system/kafka.service <<EOF >/dev/null
141+
[Unit]
142+
Description=Apache Kafka Server
143+
Documentation=http://kafka.apache.org/documentation.html
144+
Requires=zookeeper.service
145+
[Service]
146+
Type=simple
147+
Environment="JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64"
148+
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
149+
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh
150+
[Install]
151+
WantedBy=multi-user.target
152+
EOF
153+
154+
- name: Start Kafka Server
155+
run: |
156+
sudo systemctl daemon-reload
157+
sudo systemctl start zookeeper
158+
sudo systemctl start kafka
159+
160+
- uses: actions/setup-python@v2
161+
with:
162+
python-version: 3.7
163+
164+
- name: Test kafka connect upgrade
165+
run: |
166+
echo "Download kafka connect "$CI_OLD_CONNECTOR_VERSION
167+
sudo mkdir -p /usr/local/share/kafka/plugins/
168+
wget https://github.com/splunk/kafka-connect-splunk/releases/download/$CI_OLD_CONNECTOR_VERSION/splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar
169+
sudo cp splunk-kafka-connect-$CI_OLD_CONNECTOR_VERSION.jar /usr/local/share/kafka/plugins/
170+
sed -i 's/plugin\.path\=connectors\//plugin\.path\=\/usr\/local\/share\/kafka\/plugins\//' /home/circleci/repo/config/connect-distributed-quickstart.properties
171+
sed -i 's/key\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/key\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
172+
sed -i 's/value\.converter\=org\.apache\.kafka\.connect\.storage\.StringConverter/value\.converter\=org\.apache\.kafka\.connect\.json\.JsonConverter/' /home/circleci/repo/config/connect-distributed-quickstart.properties
173+
pyenv global 3.6.5
174+
pip install --upgrade pip
175+
pip install -r test/requirements.txt
176+
export PYTHONWARNINGS="ignore:Unverified HTTPS request"
177+
echo "Test kafka connect upgrade ..."
178+
python test/lib/connector_upgrade.py
179+
180+
- name: Install kafka connect
181+
run: |
182+
sudo rm -f /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar
183+
sudo cp /home/circleci/repo/target/splunk-kafka-connect*.jar /usr/local/share/kafka/plugins/
184+
test -f /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar && echo /usr/local/share/kafka/plugins/splunk-kafka-connect*.jar
185+
sudo /usr/local/kafka/bin/connect-distributed.sh /home/circleci/repo/config/connect-distributed-quickstart.properties &
186+
187+
- name: Run functional tests
188+
run: |
189+
sleep 5
190+
echo "Running functional tests....."
191+
python -m pytest -p no:warnings -s

‎ci/k8s-splunk.yml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: splunk
5+
spec:
6+
hostNetwork: true
7+
securityContext:
8+
runAsUser: 0
9+
runAsGroup: 0
10+
containers:
11+
- name: splunk
12+
image: splunk/splunk:latest
13+
ports:
14+
- containerPort: 8000
15+
hostPort: 8000
16+
protocol: TCP
17+
- containerPort: 8088
18+
hostPort: 8088
19+
protocol: TCP
20+
- containerPort: 8089
21+
hostPort: 8089
22+
protocol: TCP
23+
env:
24+
- name: SPLUNK_START_ARGS
25+
value: --accept-license
26+
- name: SPLUNK_USER
27+
value: root
28+
- name: SPLUNK_PASSWORD
29+
value: helloworld
30+
- name: SPLUNK_LAUNCH_CONF
31+
value: OPTIMISTIC_ABOUT_FILE_LOCKING=1

0 commit comments

Comments
 (0)