Skip to content
This repository was archived by the owner on Oct 22, 2024. It is now read-only.

Commit 205ffc8

Browse files
committed
tests: run device manager unit tests inside virtual machine
Device manager tests need root/sudo user permission, and loop control device on host where they run. So to avoid this we run these tests under a new virtual machine. Added new make target 'run_dm_tests' which sets up pmem-csi-${CLUSTER}-worker1 node if not exists and runs the device manager unit tests in it. To make use of existing test cluster scripts, made needed changes to them to support creating a single pmem node.
1 parent 522da48 commit 205ffc8

File tree

6 files changed

+230
-169
lines changed

6 files changed

+230
-169
lines changed

test/setup-clear-govm.sh

Lines changed: 77 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,21 @@ set -x
1010
set -o errexit # TODO: replace with explicit error checking and error messages.
1111
set -o pipefail
1212

13+
: ${INIT_KUBERNETES:=true}
1314
HOSTNAME=${HOSTNAME:-$1}
1415
IPADDR=${IPADDR:-127.0.0.1}
15-
BUNDLES="cloud-native-basic containers-basic ${TEST_CLEAR_LINUX_BUNDLES}"
16+
BUNDLES=" ${TEST_CLEAR_LINUX_BUNDLES}"
17+
if ${INIT_KUBERNETES}; then
18+
BUNDLES="${BUNDLES} cloud-native-basic containers-basic"
19+
fi
1620

1721
function error_handler(){
1822
local line="${1}"
1923
echo >&2 "ERROR: the command '${BASH_COMMAND}' at $0:${line} failed"
2024
}
25+
trap 'error_handler ${LINENO}' ERR
2126

22-
function install_kubernetes(){
23-
trap 'error_handler ${LINENO}' ERR
27+
function install_bundles(){
2428
# Setup clearlinux environment
2529
# Disable swupd autoupdate service
2630
swupd autoupdate --disable
@@ -54,100 +58,102 @@ EOF
5458
fi
5559
swapoff -a
5660

57-
# We put config changes in place for both runtimes, even though only one of them will
58-
# be used by Kubernetes, just in case that someone wants to use them manually.
61+
if ${INIT_KUBERNETES}; then
62+
# We put config changes in place for both runtimes, even though only one of them will
63+
# be used by Kubernetes, just in case that someone wants to use them manually.
5964

60-
# Proxy settings for CRI-O.
61-
mkdir /etc/systemd/system/crio.service.d
62-
cat >/etc/systemd/system/crio.service.d/proxy.conf <<EOF
65+
# Proxy settings for CRI-O.
66+
mkdir /etc/systemd/system/crio.service.d
67+
cat >/etc/systemd/system/crio.service.d/proxy.conf <<EOF
6368
[Service]
6469
Environment="HTTP_PROXY=${HTTP_PROXY}" "HTTPS_PROXY=${HTTPS_PROXY}" "NO_PROXY=${NO_PROXY}"
6570
EOF
6671

67-
# Testing may involve a Docker registry running on the build host (see
68-
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
69-
# registry, otherwise CRI-O will fail to pull images from it.
72+
# Testing may involve a Docker registry running on the build host (see
73+
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
74+
# registry, otherwise CRI-O will fail to pull images from it.
7075

71-
mkdir -p /etc/containers
72-
cat >/etc/containers/registries.conf <<EOF
76+
mkdir -p /etc/containers
77+
cat >/etc/containers/registries.conf <<EOF
7378
[registries.insecure]
7479
registries = [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ]
7580
EOF
7681

77-
# The same for Docker.
78-
mkdir -p /etc/docker
79-
cat >/etc/docker/daemon.json <<EOF
82+
# The same for Docker.
83+
mkdir -p /etc/docker
84+
cat >/etc/docker/daemon.json <<EOF
8085
{ "insecure-registries": [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ] }
8186
EOF
8287

83-
# Proxy settings for Docker.
84-
mkdir -p /etc/systemd/system/docker.service.d/
85-
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
88+
# Proxy settings for Docker.
89+
mkdir -p /etc/systemd/system/docker.service.d/
90+
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
8691
[Service]
8792
Environment="HTTP_PROXY=$HTTP_PROXY" "HTTPS_PROXY=$HTTPS_PROXY" "NO_PROXY=$NO_PROXY"
8893
EOF
8994

90-
# Disable the use of Kata containers as default runtime in Docker.
91-
# The Kubernetes control plan (apiserver, etc.) fails to run otherwise
92-
# ("Host networking requested, not supported by runtime").
95+
# Disable the use of Kata containers as default runtime in Docker.
96+
# The Kubernetes control plan (apiserver, etc.) fails to run otherwise
97+
# ("Host networking requested, not supported by runtime").
9398

94-
cat >/etc/systemd/system/docker.service.d/51-runtime.conf <<EOF
99+
cat >/etc/systemd/system/docker.service.d/51-runtime.conf <<EOF
95100
[Service]
96101
Environment="DOCKER_DEFAULT_RUNTIME=--default-runtime runc"
97102
EOF
98-
mkdir -p /etc/systemd/system/kubelet.service.d/
99-
case $TEST_CRI in
100-
docker)
101-
cri_daemon=docker
102-
# Choose Docker by disabling the use of CRI-O in KUBELET_EXTRA_ARGS.
103-
cat >/etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
103+
104+
mkdir -p /etc/systemd/system/kubelet.service.d/
105+
case $TEST_CRI in
106+
docker)
107+
cri_daemon=docker
108+
# Choose Docker by disabling the use of CRI-O in KUBELET_EXTRA_ARGS.
109+
cat >/etc/systemd/system/kubelet.service.d/10-kubeadm.conf <<EOF
104110
[Service]
105111
Environment="KUBELET_EXTRA_ARGS="
106112
EOF
107-
;;
108-
crio)
109-
cri_daemon=cri-o
110-
;;
111-
*)
112-
echo "ERROR: unsupported TEST_CRI=$TEST_CRI"
113-
exit 1
114-
;;
115-
esac
116-
117-
# kubelet must start after the container runtime that it depends on.
118-
# This is not currently configured in Clear Linux (https://github.com/clearlinux/distribution/issues/1004).
119-
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
113+
;;
114+
crio)
115+
cri_daemon=cri-o
116+
;;
117+
*)
118+
echo "ERROR: unsupported TEST_CRI=$TEST_CRI"
119+
exit 1
120+
;;
121+
esac
122+
123+
# kubelet must start after the container runtime that it depends on.
124+
# This is not currently configured in Clear Linux (https://github.com/clearlinux/distribution/issues/1004).
125+
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
120126
[Unit]
121127
After=$cri_daemon.service
122128
EOF
123-
124-
# flannel + CRI-O + Kata Containers needs a crio.conf change (https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes):
125-
# If you are using CRI-O and flannel and you want to use Kata Containers, edit the /etc/crio/crio.conf file to add:
126-
# [crio.runtime]
127-
# manage_network_ns_lifecycle = true
128-
#
129-
# We don't use Kata Containers, so that particular change is not made to /etc/crio/crio.conf
130-
# at this time.
131-
132-
# /opt/cni/bin is where runtimes like CRI-O expect CNI plugins. But cloud-native-basic installs into
133-
# /usr/libexec/cni. Instructions at https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes#id2
134-
# are inconsistent at this time (https://github.com/clearlinux/clear-linux-documentation/issues/388).
135-
#
136-
# We solve this by creating the directory and symlinking all existing CNI plugins into it.
137-
mkdir -p /opt/cni/bin
138-
for i in /usr/libexec/cni/*;do
139-
ln -s $i /opt/cni/bin/
140-
done
141-
142-
# Reconfiguration done, start daemons. Starting kubelet must wait until kubeadm has created
143-
# the necessary config files.
144-
systemctl daemon-reload
145-
systemctl restart $cri_daemon || (
146-
systemctl status $cri_daemon || true
147-
journalctl -xe || true
148-
false
149-
)
150-
systemctl enable $cri_daemon kubelet
129+
# flannel + CRI-O + Kata Containers needs a crio.conf change (https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes):
130+
# If you are using CRI-O and flannel and you want to use Kata Containers, edit the /etc/crio/crio.conf file to add:
131+
# [crio.runtime]
132+
# manage_network_ns_lifecycle = true
133+
#
134+
# We don't use Kata Containers, so that particular change is not made to /etc/crio/crio.conf
135+
# at this time.
136+
137+
# /opt/cni/bin is where runtimes like CRI-O expect CNI plugins. But cloud-native-basic installs into
138+
# /usr/libexec/cni. Instructions at https://clearlinux.org/documentation/clear-linux/tutorials/kubernetes#id2
139+
# are inconsistent at this time (https://github.com/clearlinux/clear-linux-documentation/issues/388).
140+
#
141+
# We solve this by creating the directory and symlinking all existing CNI plugins into it.
142+
mkdir -p /opt/cni/bin
143+
for i in /usr/libexec/cni/*;do
144+
ln -s $i /opt/cni/bin/
145+
done
146+
147+
# Reconfiguration done, start daemons. Starting kubelet must wait until kubeadm has created
148+
# the necessary config files.
149+
systemctl daemon-reload
150+
systemctl restart $cri_daemon || (
151+
systemctl status $cri_daemon || true
152+
journalctl -xe || true
153+
false
154+
)
155+
systemctl enable $cri_daemon kubelet
156+
fi
151157
}
152158

153-
install_kubernetes
159+
install_bundles

test/setup-fedora-govm.sh

Lines changed: 52 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ set -x
1010
set -o errexit # TODO: replace with explicit error checking and error messages.
1111
set -o pipefail
1212

13+
: ${INIT_KUBERNETES:=true}
1314
HOSTNAME=${HOSTNAME:-$1}
1415
IPADDR=${IPADDR:-127.0.0.1}
1516

@@ -19,29 +20,32 @@ function error_handler(){
1920
}
2021
trap 'error_handler ${LINENO}' ERR
2122

22-
# Always use Docker, and always use the same version for reproducibility.
23-
cat <<'EOF' > /etc/yum.repos.d/docker-ce.repo
23+
24+
# For PMEM.
25+
packages+=" ndctl"
26+
27+
# Some additional utilities.
28+
packages+=" device-mapper-persistent-data lvm2"
29+
30+
if ${INIT_KUBERNETES}; then
31+
# Always use Docker, and always use the same version for reproducibility.
32+
cat <<'EOF' > /etc/yum.repos.d/docker-ce.repo
2433
[docker-ce-stable]
2534
name=Docker CE Stable - $basearch
2635
baseurl=https://download.docker.com/linux/centos/7/$basearch/stable
2736
enabled=1
2837
gpgcheck=1
2938
gpgkey=https://download.docker.com/linux/centos/gpg
3039
EOF
31-
packages+=" docker-ce-3:19.03.2-3.el7"
40+
packages+=" docker-ce-3:19.03.2-3.el7"
3241

33-
# For PMEM.
34-
packages+=" ndctl"
35-
36-
# Some additional utilities.
37-
packages+=" device-mapper-persistent-data lvm2"
42+
# Install according to https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
43+
modprobe br_netfilter
44+
echo 1 >/proc/sys/net/bridge/bridge-nf-call-iptables
45+
setenforce 0
46+
sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
3847

39-
# Install according to https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
40-
modprobe br_netfilter
41-
echo 1 >/proc/sys/net/bridge/bridge-nf-call-iptables
42-
setenforce 0
43-
sed -i --follow-symlinks 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/sysconfig/selinux
44-
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
48+
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
4549
[kubernetes]
4650
name=Kubernetes
4751
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
@@ -51,17 +55,18 @@ repo_gpgcheck=1
5155
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
5256
EOF
5357

54-
# For the sake of reproducibility, use fixed versions.
55-
# List generated with:
56-
# for v in 1.13 1.14 1.15 1.16; do for i in kubelet kubeadm kubectl; do echo "$i-$(sudo yum --showduplicates list kubelet | grep " $v" | sed -e 's/.* \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/' | sort -u | tail -n 1)"; done; done
57-
case ${TEST_KUBERNETES_VERSION} in
58-
1.13) packages+=" kubelet-1.13.9-0 kubeadm-1.13.9-0 kubectl-1.13.9-0";;
59-
1.14) packages+=" kubelet-1.14.7-0 kubeadm-1.14.7-0 kubectl-1.14.7-0";;
60-
1.15) packages+=" kubelet-1.15.4-0 kubeadm-1.15.4-0 kubectl-1.15.4-0";;
61-
1.16) packages+=" kubelet-1.16.0-0 kubeadm-1.16.0-0 kubectl-1.16.0-0";;
62-
*) echo >&2 "Kubernetes version ${TEST_KUBERNETES_VERSION} not supported, package list in $0 must be updated."; exit 1;;
63-
esac
64-
packages+=" --disableexcludes=kubernetes"
58+
# For the sake of reproducibility, use fixed versions.
59+
# List generated with:
60+
# for v in 1.13 1.14 1.15 1.16; do for i in kubelet kubeadm kubectl; do echo "$i-$(sudo yum --showduplicates list kubelet | grep " $v" | sed -e 's/.* \([0-9]*\.[0-9]*\.[0-9]*[^ ]*\).*/\1/' | sort -u | tail -n 1)"; done; done
61+
case ${TEST_KUBERNETES_VERSION} in
62+
1.13) packages+=" kubelet-1.13.9-0 kubeadm-1.13.9-0 kubectl-1.13.9-0";;
63+
1.14) packages+=" kubelet-1.14.7-0 kubeadm-1.14.7-0 kubectl-1.14.7-0";;
64+
1.15) packages+=" kubelet-1.15.4-0 kubeadm-1.15.4-0 kubectl-1.15.4-0";;
65+
1.16) packages+=" kubelet-1.16.0-0 kubeadm-1.16.0-0 kubectl-1.16.0-0";;
66+
*) echo >&2 "Kubernetes version ${TEST_KUBERNETES_VERSION} not supported, package list in $0 must be updated."; exit 1;;
67+
esac
68+
packages+=" --disableexcludes=kubernetes"
69+
fi
6570

6671
# Sometimes we hit a bad mirror and get "Failed to synchronize cache for repo ...".
6772
# https://unix.stackexchange.com/questions/487635/fedora-29-failed-to-synchronize-cache-for-repo-fedora-modular
@@ -81,34 +86,36 @@ while ! yum install -y $packages; do
8186
fi
8287
done
8388

84-
# Upstream kubelet looks in /opt/cni/bin, actual files are in
85-
# /usr/libexec/cni from
86-
# containernetworking-plugins-0.8.1-1.fc30.x86_64.
87-
mkdir -p /opt/cni
88-
ln -s /usr/libexec/cni /opt/cni/bin
89-
90-
# Testing may involve a Docker registry running on the build host (see
91-
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
92-
# registry, otherwise Docker will fail to pull images from it.
93-
mkdir -p /etc/docker
94-
cat >/etc/docker/daemon.json <<EOF
89+
if $INIT_KUBERNETES; then
90+
# Upstream kubelet looks in /opt/cni/bin, actual files are in
91+
# /usr/libexec/cni from
92+
# containernetworking-plugins-0.8.1-1.fc30.x86_64.
93+
mkdir -p /opt/cni
94+
ln -s /usr/libexec/cni /opt/cni/bin
95+
96+
# Testing may involve a Docker registry running on the build host (see
97+
# TEST_LOCAL_REGISTRY and TEST_PMEM_REGISTRY). We need to trust that
98+
# registry, otherwise Docker will fail to pull images from it.
99+
mkdir -p /etc/docker
100+
cat >/etc/docker/daemon.json <<EOF
95101
{ "insecure-registries": [ $(echo $INSECURE_REGISTRIES | sed 's|^|"|g;s| |", "|g;s|$|"|') ] }
96102
EOF
97103

98-
# Proxy settings for Docker.
99-
mkdir -p /etc/systemd/system/docker.service.d/
100-
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
104+
# Proxy settings for Docker.
105+
mkdir -p /etc/systemd/system/docker.service.d/
106+
cat >/etc/systemd/system/docker.service.d/proxy.conf <<EOF
101107
[Service]
102108
Environment="HTTP_PROXY=$HTTP_PROXY" "HTTPS_PROXY=$HTTPS_PROXY" "NO_PROXY=$NO_PROXY"
103109
EOF
104110

105-
# kubelet must start after the container runtime that it depends on.
106-
mkdir -p /etc/systemd/system/kubelet.service.d
107-
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
111+
# kubelet must start after the container runtime that it depends on.
112+
mkdir -p /etc/systemd/system/kubelet.service.d
113+
cat >/etc/systemd/system/kubelet.service.d/10-cri.conf <<EOF
108114
[Unit]
109115
After=docker.service
110116
EOF
111117

112-
update-alternatives --set iptables /usr/sbin/iptables-legacy
113-
systemctl daemon-reload
114-
systemctl enable --now docker kubelet
118+
update-alternatives --set iptables /usr/sbin/iptables-legacy
119+
systemctl daemon-reload
120+
systemctl enable --now docker kubelet
121+
fi

test/setup-kubernetes.sh

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ set -x
99
set -o errexit # TODO: replace with explicit error checking and messages
1010
set -o pipefail
1111

12-
: ${TEST_INIT_REGION:=true}
1312
: ${TEST_CREATE_REGISTRY:=false}
1413

1514
function error_handler(){
@@ -175,19 +174,6 @@ ${TEST_CONFIGURE_POST_ALL}
175174

176175
}
177176

178-
179-
function init_region(){
180-
trap 'error_handler ${LINENO}' ERR
181-
sudo ndctl disable-region region0
182-
sudo ndctl init-labels nmem0
183-
sudo ndctl enable-region region0
184-
185-
}
186-
187-
if $TEST_INIT_REGION; then
188-
init_region
189-
fi
190-
191177
if [[ "$HOSTNAME" == *"master"* ]]; then
192178
setup_kubernetes_master
193179
if $TEST_CREATE_REGISTRY; then

0 commit comments

Comments
 (0)