-
Notifications
You must be signed in to change notification settings - Fork 108
228 lines (196 loc) · 8.03 KB
/
harbor-e2e-tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
name: Harbor E2E Tests
on:
push:
branches:
- '**'
pull_request:
branches:
- main
- release-*
- feature/*
jobs:
harbor-tests:
runs-on: ubuntu-latest
name: K8S v${{ matrix.k8sVersion }} (CM v${{ matrix.certManager }} ${{ matrix.samples }})
env:
USE_EXISTING_CLUSTER: true
operatorNamespace: harbor-operator-ns
dockerImage: harbor-operator:dev_test
strategy:
fail-fast: false
matrix:
# https://github.com/jetstack/cert-manager/tags
certManager:
- "1.6.3"
- "1.9.1"
# https://github.com/kubernetes-sigs/kind/releases
k8sVersion:
- "1.21.12"
- "1.23.6"
- "1.24.0"
# https://github.com/kubernetes/ingress-nginx/tags
ingress:
- "1.3.0"
samples:
- "full_stack.yaml"
- "standard_stack_fs.yaml"
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: 1.18
- uses: azure/setup-kubectl@v3
with:
version: 'latest'
- name: Cache go mod
uses: actions/cache@v2
with:
path: ~/go/pkg/mod
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-go-
- name: Prepare memory storage for etcd of kind cluster
run: |
# Use memory storage for etcd of the kind cluster, see https://github.com/kubernetes-sigs/kind/issues/845 for more info
mkdir -p /tmp/lib/etcd
sudo mount -t tmpfs tmpfs /tmp/lib/etcd
- name: Install Kubernetes v${{ matrix.k8sVersion }}
uses: helm/kind-action@v1.3.0
with:
version: v0.14.0
node_image: kindest/node:v${{ matrix.k8sVersion }}
cluster_name: harbor
config: .github/kind_permission.yaml
- name: Install CertManager v${{ matrix.certManager }}
run: |
kubectl apply -f "https://github.com/jetstack/cert-manager/releases/download/v${{ matrix.certManager }}/cert-manager.yaml"
sleep 5
time kubectl -n cert-manager wait --for=condition=Available deployment --all --timeout 300s
- name: Install Ingress
run: |
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${{ matrix.ingress }}/deploy/static/provider/kind/deploy.yaml
time kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=300s
- name: build harbor-operator
run: |
docker buildx create --use
make generate docker-build IMG=${dockerImage} GIT_COMMIT=${{ github.sha }}
kind load docker-image ${dockerImage} --name harbor
- name: install harbor-operator
run: |
make kustomize
cd manifests/cluster
kustomize edit add secret github-token --disableNameSuffixHash --from-literal=GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
kustomize edit add patch --path patch/github-token.yaml
kustomize edit set image goharbor/harbor-operator=${dockerImage}
../../bin/kustomize build --reorder legacy | kubectl create -f -
if ! time kubectl -n ${operatorNamespace} wait --for=condition=Available deployment --all --timeout 300s; then
kubectl get all -n ${operatorNamespace}
exit 1
fi
kubectl get all -n ${operatorNamespace}
sleep 10
- name: install harbor
run: |
set -ex
IP=`hostname -I | awk '{print $1}'`
echo "IP=$IP" >> $GITHUB_ENV
CORE_HOST=core.$IP.nip.io
NOTARY_HOST=notary.$IP.nip.io
MINIO_HOST=minio.$IP.nip.io
echo "CORE_HOST=$CORE_HOST" >> $GITHUB_ENV
echo "NOTARY_HOST=$NOTARY_HOST" >> $GITHUB_ENV
echo "MINIO_HOST=$MINIO_HOST" >> $GITHUB_ENV
samplefile=${{ matrix.samples }}
sed -i "s/core.harbor.domain/$CORE_HOST/g" manifests/samples/$samplefile
sed -i "s/notary.harbor.domain/$NOTARY_HOST/g" manifests/samples/$samplefile
sed -i "s/minio.harbor.domain/$MINIO_HOST/g" manifests/samples/$samplefile
sed -i "s/logLevel: info/logLevel: debug/g" manifests/samples/$samplefile
kubectl apply -f manifests/samples/$samplefile
for i in $(seq 1 7);do
sleep 30
echo $i
kubectl -n cluster-sample-ns get all
done
function wait-for-condition () {
time kubectl -n cluster-sample-ns wait --for=condition=$1 harborcluster harborcluster-sample --timeout $2
}
if ! wait-for-condition InProgress=False 600s && ! wait-for-condition Failed=False 60s; then
echo install harbor failed
kubectl describe harborcluster -n cluster-sample-ns
kubectl describe harbor -n cluster-sample-ns
kubectl get all -n cluster-sample-ns
for n in $(kubectl -n cluster-sample-ns get po |grep -v Running|grep -v NAME|awk '{print $1}');do
echo describe $n
kubectl -n cluster-sample-ns describe pod $n
echo show log $n
kubectl -n cluster-sample-ns logs --tail 100 $n || true
done
kubectl logs -l control-plane=harbor-operator -n ${operatorNamespace} --tail 100
free -h
exit 1
else
kubectl -n cluster-sample-ns get all -o wide
kubectl get harbor -n cluster-sample-ns -o wide
kubectl get harborcluster -n cluster-sample-ns -o wide
kubectl get all -n ${operatorNamespace}
fi
free -h
df -h
- name: test harbor
run: |
set -ex
free -h
df -h
curl https://$CORE_HOST/api/v2.0/systeminfo -i -k -f
sudo mkdir -p /etc/docker/certs.d/$CORE_HOST
kubectl -n cluster-sample-ns get secret sample-public-certificate -o jsonpath='{.data.ca\.crt}' \
| base64 --decode \
| sudo tee /etc/docker/certs.d/$CORE_HOST/harbor_ca.crt
# docker login, create image, docker push, docker pull
docker login $CORE_HOST -u admin -p Harbor12345 || (kubectl -n cluster-sample-ns get po;kubectl -n cluster-sample-ns logs -l goharbor.io/operator-controller=core;exit 1)
docker run busybox dd if=/dev/urandom of=test count=10 bs=1MB
DOCKERID=`docker ps -l -q`
docker commit $DOCKERID $CORE_HOST/library/busybox:test
docker push $CORE_HOST/library/busybox:test
docker pull $CORE_HOST/library/busybox:test
- name: apidb test
run: bash .github/scripts/apidb_test.sh
env:
DOCKER_USER: ${{ secrets.DOCKER_USER }}
DOCKER_PWD: ${{ secrets.DOCKER_TOKEN }}
CORE_DEPLOYMENT: harborcluster-sample-harbor-harbor-core
NAMESPACE: cluster-sample-ns
- name: fetch harbor logs
if: ${{ failure() }}
run: |
df -h
free -m
mkdir -p /tmp/harbor
for name in core jobservice registry registryctl trivy chartmuseum notaryserver notarysigner portal; do \
kubectl -n cluster-sample-ns logs -l "goharbor.io/operator-controller=$name" --all-containers > /tmp/harbor/$name.log ; \
done
kubectl -n cluster-sample-ns logs -l "application=spilo" --all-containers > /tmp/harbor/db.log
kubectl -n cluster-sample-ns logs -l "app.kubernetes.io/component=redis" --all-containers > /tmp/harbor/redis.log
ls -l /tmp/harbor
- uses: actions/upload-artifact@v2
if: ${{ failure() }}
with:
name: harbor_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
path: /tmp/harbor
- name: fetch logs
if: ${{ failure() }}
run: |
mkdir -p /tmp/logs
kind export logs --name harbor /tmp/logs
ls -l /tmp/logs
- uses: actions/upload-artifact@v2
if: ${{ failure() }}
with:
name: kind_v${{ matrix.k8sVersion }}_v${{ matrix.certManager }}_${{ matrix.samples }}
path: /tmp/logs
- name: Get logs for debug
if: ${{ failure() }}
run: |
set -x
kubectl get all -n "${operatorNamespace}" -o wide
kubectl logs -n "${operatorNamespace}" -l 'control-plane=harbor-operator' --all-containers --tail=1000