From a8843801a42a1d5a3c2e03c56439995a1105b780 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 6 Feb 2018 11:47:38 +0000 Subject: [PATCH 1/4] test: Basic build tests for all 4 combinations of runtime/network. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit {docker,cri} × {bridge,weave} Needs a top-level Makefile hook to control the output directory. Signed-off-by: Ian Campbell --- Makefile | 6 ++-- test/.gitignore | 2 ++ test/cases/000_smoke/001_cri-bridge/test.sh | 9 +++++ test/cases/000_smoke/002_cri-weave/test.sh | 9 +++++ .../cases/000_smoke/003_docker-bridge/test.sh | 9 +++++ test/cases/000_smoke/004_docker-weave/test.sh | 9 +++++ test/cases/000_smoke/common.sh | 26 ++++++++++++++ test/cases/000_smoke/group.sh | 31 +++++++++++++++++ test/cases/group.sh | 34 +++++++++++++++++++ 9 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 test/.gitignore create mode 100755 test/cases/000_smoke/001_cri-bridge/test.sh create mode 100755 test/cases/000_smoke/002_cri-weave/test.sh create mode 100755 test/cases/000_smoke/003_docker-bridge/test.sh create mode 100755 test/cases/000_smoke/004_docker-weave/test.sh create mode 100644 test/cases/000_smoke/common.sh create mode 100755 test/cases/000_smoke/group.sh create mode 100755 test/cases/group.sh diff --git a/Makefile b/Makefile index e21cc03..43c02f7 100644 --- a/Makefile +++ b/Makefile @@ -11,14 +11,16 @@ endif KUBE_FORMAT_ARGS := $(patsubst %,-format %,$(KUBE_FORMATS)) +KUBE_BASENAME ?= kube- + .PHONY: all master node all: master node master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml - linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-master $(KUBE_FORMAT_ARGS) $^ + linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)master $(KUBE_FORMAT_ARGS) $^ node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml - linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name kube-node $(KUBE_FORMAT_ARGS) $^ + linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)node $(KUBE_FORMAT_ARGS) $^ yml/weave.yml: kube-weave.yaml diff --git a/test/.gitignore b/test/.gitignore new file mode 100644 index 0000000..d99da00 --- /dev/null +++ b/test/.gitignore @@ -0,0 +1,2 @@ +_results +cases/_tmp diff --git a/test/cases/000_smoke/001_cri-bridge/test.sh b/test/cases/000_smoke/001_cri-bridge/test.sh new file mode 100755 index 0000000..0b826c1 --- /dev/null +++ b/test/cases/000_smoke/001_cri-bridge/test.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# SUMMARY: build and boot using cri-containerd runtime and Bridged networking +# LABELS: + +runtime=cri-containerd +network=bridge + +# Doesn't return +. ../common.sh diff --git a/test/cases/000_smoke/002_cri-weave/test.sh b/test/cases/000_smoke/002_cri-weave/test.sh new file mode 100755 index 0000000..e52d3b8 --- /dev/null +++ b/test/cases/000_smoke/002_cri-weave/test.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# SUMMARY: build and boot using cri-containerd runtime and Weave networking +# LABELS: + +runtime=cri-containerd +network=weave + +# Doesn't return +. ../common.sh diff --git a/test/cases/000_smoke/003_docker-bridge/test.sh b/test/cases/000_smoke/003_docker-bridge/test.sh new file mode 100755 index 0000000..84b098b --- /dev/null +++ b/test/cases/000_smoke/003_docker-bridge/test.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# SUMMARY: build and boot using Docker runtime and Bridged networking +# LABELS: + +runtime=docker +network=bridge + +# Doesn't return +. ../common.sh diff --git a/test/cases/000_smoke/004_docker-weave/test.sh b/test/cases/000_smoke/004_docker-weave/test.sh new file mode 100755 index 0000000..43231a9 --- /dev/null +++ b/test/cases/000_smoke/004_docker-weave/test.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# SUMMARY: build and boot using Docker runtime and Weave networking +# LABELS: + +runtime=docker +network=weave + +# Doesn't return +. ../common.sh diff --git a/test/cases/000_smoke/common.sh b/test/cases/000_smoke/common.sh new file mode 100644 index 0000000..bb75a52 --- /dev/null +++ b/test/cases/000_smoke/common.sh @@ -0,0 +1,26 @@ +# To be sourced by */test.sh + +set -e + +# Source libraries. Uncomment if needed/defined +#. "${RT_LIB}" +#. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +if [ "x$runtime" = "x" ] || [ "x$network" = "x" ] ; then + echo "common.sh requires \$runtime and \$network" >&2 + exit 1 +fi + +clean_up() { + rm -f kube-master.iso +} + +trap clean_up EXIT + +export KUBE_RUNTIME=$runtime +export KUBE_NETWORK=$network +export LINUXKIT_BUILD_ARGS="--disable-content-trust" +export KUBE_BASENAME="`pwd`/kube-" +make -C ${RT_PROJECT_ROOT}/../../ master + +exit 0 diff --git a/test/cases/000_smoke/group.sh b/test/cases/000_smoke/group.sh new file mode 100755 index 0000000..4ceea7f --- /dev/null +++ b/test/cases/000_smoke/group.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# SUMMARY: Basic build and run smoke tests +# LABELS: + +group_init() { + # Group initialisation code goes here + return 0 +} + +group_deinit() { + # Group de-initialisation code goes here + return 0 +} + +CMD=$1 +case $CMD in +init) + group_init + res=$? + ;; +deinit) + group_deinit + res=$? + ;; +*) + res=1 + ;; +esac + +exit $res + diff --git a/test/cases/group.sh b/test/cases/group.sh new file mode 100755 index 0000000..fa58b25 --- /dev/null +++ b/test/cases/group.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# NAME: kubernetes +# SUMMARY: LinuxKit/Kubernetes regression tests + +# Source libraries. Uncomment if needed/defined +# . "${RT_LIB}" +#. "${RT_PROJECT_ROOT}/_lib/lib.sh" + +group_init() { + # Group initialisation code goes here + return 0 +} + +group_deinit() { + # Group de-initialisation code goes here + return 0 +} + +CMD=$1 +case $CMD in +init) + group_init + res=$? + ;; +deinit) + group_deinit + res=$? + ;; +*) + res=1 + ;; +esac + +exit $res From 798d14d099ea3f891887a2d4c147dff4ec0e895e Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 7 Feb 2018 12:21:19 +0000 Subject: [PATCH 2/4] test: Boot and initialise the cluster Signed-off-by: Ian Campbell --- test/cases/000_smoke/common.sh | 3 + test/cases/000_smoke/test.exp | 211 +++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100755 test/cases/000_smoke/test.exp diff --git a/test/cases/000_smoke/common.sh b/test/cases/000_smoke/common.sh index bb75a52..5a7cf87 100644 --- a/test/cases/000_smoke/common.sh +++ b/test/cases/000_smoke/common.sh @@ -13,6 +13,7 @@ fi clean_up() { rm -f kube-master.iso + rm -rf kube-master-state } trap clean_up EXIT @@ -23,4 +24,6 @@ export LINUXKIT_BUILD_ARGS="--disable-content-trust" export KUBE_BASENAME="`pwd`/kube-" make -C ${RT_PROJECT_ROOT}/../../ master +../test.exp ${RT_PROJECT_ROOT}/../../boot.sh ${RT_PROJECT_ROOT}/../../ssh_into_kubelet.sh + exit 0 diff --git a/test/cases/000_smoke/test.exp b/test/cases/000_smoke/test.exp new file mode 100755 index 0000000..7058374 --- /dev/null +++ b/test/cases/000_smoke/test.exp @@ -0,0 +1,211 @@ +#!/usr/bin/env expect + +set con_prompt "(ns: getty) linuxkit-*:*# " +set ssh_prompt "linuxkit-*:*# " +set timeout 120 + +set bootexec [lindex $argv 0] +set sshexec [lindex $argv 1] + +proc kill args { + foreach what $args { + global $what + if [info exists $what] { + upvar #0 $what sid + set pid [exp_pid -i $sid] + puts "killing $what ($sid) = $pid" + exec kill $pid + close $sid + } else { + puts "not killing $what (not started)" + } + } +} + +proc boot_linuxkit {} { + global lk_sid bootexec + spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y $bootexec + set lk_sid $spawn_id + puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]" +} + +proc ssh_into_kubelet {} { + global ssh_sid sshexec + + set sshopts {-p 2222 -o ConnectTimeout=5 -o LogLevel=DEBUG} + if [info exists ::env(SSHOPTS)] { + set sshopts [concat $::env(SSHOPTS) $sshopts] + } + spawn env SSHOPTS=$sshopts $sshexec localhost + set ssh_sid $spawn_id + puts "INFO ssh ($ssh_sid) is pid [exp_pid -i $ssh_sid]" +} + +proc await_prompt {sidvar promptvar step} { + upvar #0 $sidvar sid $promptvar prompt + expect -i $sid \ + $prompt { + puts "SUCCESS $step" + } timeout { + puts "FAIL $step (timeout)" + kill ssh_sid lk_sid + exit 1 + } eof { + puts "FAIL $step (eof)" + kill ssh_sid lk_sid + exit 1 + } +} + +proc send_concmd {s} { + global lk_sid + send -i $lk_sid "$s\n" +} + +proc await_con_prompt {step} { + global lk_sid con_prompt + await_prompt lk_sid con_prompt $step +} + +proc concmd {step cmd} { + send_concmd $cmd + await_con_prompt $step +} + +proc send_sshcmd {s} { + global ssh_sid + send -i $ssh_sid "$s\n" +} + +proc await_ssh_prompt {step} { + global ssh_sid ssh_prompt + await_prompt ssh_sid ssh_prompt $step +} + +proc sshcmd {step cmd} { + send_sshcmd $cmd + await_ssh_prompt $step +} + +# Run $cmd at 1s intervals until we get $want or timeout +# $sidvar names a variable containing sid to send/receive on +# $promptvar names a variable containing the expected prompt +# $step is used for logging +proc await_cmd_output {sidvar promptvar step cmd want} { + upvar #0 $sidvar sid $promptvar prompt + send -i $sid "$cmd\n" + set retries 0 + expect -i $sid -timeout 300 \ + $want { + puts "SUCCESS $step: $cmd => $want" + } $prompt { + set retries [expr $retries + 1] + sleep 1 + send -i $sid "$cmd\n" + exp_continue -continue_timer + } timeout { + puts "FAIL $step (timeout)" + kill ssh_sid lk_sid + exit 1 + } + await_prompt $sidvar $promptvar "$step (after $retries attempts)" +} + +proc await_sshcmd_output {step cmd want} { + global ssh_sid ssh_prompt + await_cmd_output ssh_sid ssh_prompt $step $cmd $want +} + +boot_linuxkit + +await_con_prompt "boot" + +concmd "ifconfig" "ifconfig eth0" +concmd "list containers" "ctr -n services.linuxkit container ls" + +set retries 0 +ssh_into_kubelet +# provide ssh_sid as an indirect, allowing ssh to be respawned, which +# changes the id, we need this in case ssh cannot immediately connect. +expect -i ssh_sid \ + $ssh_prompt { + puts "SUCCESS connected to ssh (after $retries attempts)" + } "read: Connection reset by peer" { + # ssh happened too soon, wait a bit. + set retries [expr $retries + 1] + #puts "RETRY:$retries ssh (conn reset)" + wait -i $ssh_sid + sleep 1 + ssh_into_kubelet + exp_continue -continue_timer + } eof { + set retries [expr $retries + 1] + #puts "RETRY:$retries ssh (eof)" + wait -i $ssh_sid + sleep 1 + ssh_into_kubelet + exp_continue -continue_timer + } timeout { + puts "FAIL ssh (timeout)" + kill ssh_sid lk_sid + exit 1 + } + +puts "RUN kubeadm-init.sh" +send_sshcmd "kubeadm-init.sh" + +# Written as N*5m with logging to avoid "10 mins with no output" from CI +set retries 0 +set maxretries 10 +expect -i $ssh_sid -timeout 300 \ + "Your Kubernetes master has initialized successfully!" { + puts "SUCCESS cluster initialised!" + } $ssh_prompt { + puts "FAIL kubeadm-init.sh (returned to prompt)" + kill ssh_sid lk_sid + exit 1 + } timeout { + set retries [expr $retries + 1] + if [expr $retries < $maxretries] { + #puts "RETRY:$retries kubeadm-init.sh (timeout)" + exp_continue + } + puts "FAIL kubeadm-init.sh (timeout)" + kill ssh_sid lk_sid + exit 1 + } eof { + puts "FAIL kubeadm-init.sh (eof)" + kill ssh_sid lk_sid + exit 1 + } +await_ssh_prompt "kubeadm-init.sh" + +sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods" + +await_sshcmd_output "await node ready" \ + {kubectl get nodes -o jsonpath='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}' ; echo} \ + "Ready=True" + +await_sshcmd_output "await kube-dns ready" \ + {kubectl --namespace=kube-system get --selector='k8s-app'='kube-dns' -o jsonpath='{.items[*].status.phase}' pods ; echo} \ + "Running" +await_sshcmd_output "await etcd ready" \ + {kubectl --namespace=kube-system get --selector='component'='etcd' -o jsonpath='{.items[*].status.phase}' pods ; echo} \ + "Running" + +sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods" + +kill ssh_sid + +puts "RUN poweroff -f" +send_concmd "poweroff -f" + +expect -i $lk_sid \ + "Power down" { + puts "SUCCESS poweroff" + } eof { + puts "SUCCESS poweroff" + } timeout { + puts "FAILED poweroff (timeout)" + exit 1 + } From d796098aa6d5c78fd72c4abdce6e7f56f08ac316 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 7 Feb 2018 12:27:40 +0000 Subject: [PATCH 3/4] test: Add a basic Deployment+Service, check access from host. Needs a top-level Makefile hook to add a new yml to the image. Signed-off-by: Ian Campbell --- Makefile | 4 +-- test/cases/000_smoke/common.sh | 1 + test/cases/000_smoke/test.exp | 20 +++++++++++++- test/cases/000_smoke/test.yml | 49 ++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 test/cases/000_smoke/test.yml diff --git a/Makefile b/Makefile index 43c02f7..61b0698 100644 --- a/Makefile +++ b/Makefile @@ -16,10 +16,10 @@ KUBE_BASENAME ?= kube- .PHONY: all master node all: master node -master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml +master: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_RUNTIME)-master.yml yml/$(KUBE_NETWORK).yml $(KUBE_EXTRA_YML) linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)master $(KUBE_FORMAT_ARGS) $^ -node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml +node: yml/kube.yml yml/$(KUBE_RUNTIME).yml yml/$(KUBE_NETWORK).yml $(KUBE_EXTRA_YML) linuxkit $(LINUXKIT_ARGS) build $(LINUXKIT_BUILD_ARGS) -name $(KUBE_BASENAME)node $(KUBE_FORMAT_ARGS) $^ yml/weave.yml: kube-weave.yaml diff --git a/test/cases/000_smoke/common.sh b/test/cases/000_smoke/common.sh index 5a7cf87..499362b 100644 --- a/test/cases/000_smoke/common.sh +++ b/test/cases/000_smoke/common.sh @@ -22,6 +22,7 @@ export KUBE_RUNTIME=$runtime export KUBE_NETWORK=$network export LINUXKIT_BUILD_ARGS="--disable-content-trust" export KUBE_BASENAME="`pwd`/kube-" +export KUBE_EXTRA_YML="`pwd`/../test.yml" make -C ${RT_PROJECT_ROOT}/../../ master ../test.exp ${RT_PROJECT_ROOT}/../../boot.sh ${RT_PROJECT_ROOT}/../../ssh_into_kubelet.sh diff --git a/test/cases/000_smoke/test.exp b/test/cases/000_smoke/test.exp index 7058374..aa3a450 100755 --- a/test/cases/000_smoke/test.exp +++ b/test/cases/000_smoke/test.exp @@ -24,7 +24,7 @@ proc kill args { proc boot_linuxkit {} { global lk_sid bootexec - spawn env {KUBE_RUN_ARGS=-publish 2222:22} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y $bootexec + spawn env {KUBE_RUN_ARGS=-publish 2222:22 -publish 30002:30002} KUBE_MEM=3584 KUBE_VCPUS=2 KUBE_DISK=8G KUBE_CLEAR_STATE=y KUBE_MASTER_UNTAINT=y $bootexec set lk_sid $spawn_id puts "INFO lk ($lk_sid) is pid [exp_pid -i $lk_sid]" } @@ -195,6 +195,24 @@ await_sshcmd_output "await etcd ready" \ sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods" +sshcmd "apply app.yaml" "kubectl apply -f /root/app.yaml" + +await_sshcmd_output "await nginx pod ready" \ + {kubectl get pods --selector=name=nginx -o jsonpath='{.items[*].status.phase}' ; echo} \ + "Running" +puts "SUCCESS application pod ready" + +sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods ; echo ; kubectl --namespace=default get -o wide pods" + +set curl [exec curl -sS http://localhost:30002] +if [string match "Welcome to nginx!" $curl] { + puts "FAIL nginx returned:" + puts $curl + kill ssh_sid lk_sid + exit 1 +} +puts "SUCCESS nginx responded well" + kill ssh_sid puts "RUN poweroff -f" diff --git a/test/cases/000_smoke/test.yml b/test/cases/000_smoke/test.yml new file mode 100644 index 0000000..3b81221 --- /dev/null +++ b/test/cases/000_smoke/test.yml @@ -0,0 +1,49 @@ +files: + - path: containers/services/kubelet/lower/root/app.yaml + contents: | + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: nginx + labels: + name: nginx + namespace: default + spec: + replicas: 1 + template: + metadata: + labels: + name: nginx + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - name: www + containerPort: 80 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + volumes: + - name: tmp-volume + emptyDir: + medium: Memory + nodeSelector: + beta.kubernetes.io/os: linux + --- + apiVersion: v1 + kind: Service + metadata: + name: nginx + labels: + name: nginx + namespace: default + spec: + type: NodePort + ports: + # the port that this service should serve on + - port: 80 + targetPort: 80 + nodePort: 30002 + selector: + name: nginx From f4cd64922c204754a9d9565c7fbc03ea6c0a5dd3 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 7 Feb 2018 13:08:37 +0000 Subject: [PATCH 4/4] test: Test intra-pod networking is ok Add an alpine pod and use it to fetch from the nginx, which ensures that network traffic as well as DNS and such are working. Signed-off-by: Ian Campbell --- test/cases/000_smoke/test.exp | 24 +++++++++++++++++++++++- test/cases/000_smoke/test.yml | 21 +++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/test/cases/000_smoke/test.exp b/test/cases/000_smoke/test.exp index aa3a450..144ad8e 100755 --- a/test/cases/000_smoke/test.exp +++ b/test/cases/000_smoke/test.exp @@ -197,10 +197,13 @@ sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-sys sshcmd "apply app.yaml" "kubectl apply -f /root/app.yaml" +await_sshcmd_output "await alpine pod ready" \ + {kubectl get pods --selector=name=alpine -o jsonpath='{.items[*].status.phase}' ; echo} \ + "Running" await_sshcmd_output "await nginx pod ready" \ {kubectl get pods --selector=name=nginx -o jsonpath='{.items[*].status.phase}' ; echo} \ "Running" -puts "SUCCESS application pod ready" +puts "SUCCESS application pods ready" sshcmd "status" "kubectl get -o wide nodes ; echo ; kubectl --namespace=kube-system get -o wide pods ; echo ; kubectl --namespace=default get -o wide pods" @@ -213,6 +216,25 @@ if [string match "Welcome to nginx!" $curl] { } puts "SUCCESS nginx responded well" +send_sshcmd {kubectl exec $(kubectl get pods -l name==alpine -o=jsonpath='{.items[*].metadata.name}') -- wget -q -O - http://nginx/} +expect -i $ssh_sid -timeout 10 \ + "Welcome to nginx!" { + puts "SUCCESS intra-pod networking ok" + } $ssh_prompt { + puts "FAIL intra-pod networking (returned to prompt)" + kill ssh_sid lk_sid + exit 1 + } timeout { + puts "FAIL intra-pod networking (timeout)" + kill ssh_sid lk_sid + exit 1 + } eof { + puts "FAIL intra-pod networking (eof)" + kill ssh_sid lk_sid + exit 1 + } +await_ssh_prompt "intra-pod networking" + kill ssh_sid puts "RUN poweroff -f" diff --git a/test/cases/000_smoke/test.yml b/test/cases/000_smoke/test.yml index 3b81221..12b64e9 100644 --- a/test/cases/000_smoke/test.yml +++ b/test/cases/000_smoke/test.yml @@ -47,3 +47,24 @@ files: nodePort: 30002 selector: name: nginx + --- + apiVersion: extensions/v1beta1 + kind: Deployment + metadata: + name: alpine + labels: + name: alpine + namespace: default + spec: + replicas: 1 + template: + metadata: + labels: + name: alpine + spec: + containers: + - name: alpine + image: alpine:3.7 + command: ["/bin/sh", "-c", "while : ; do sleep 1h ; done"] + nodeSelector: + beta.kubernetes.io/os: linux