chore: move bash tests to integration

move extensions and secureboot tests to integration. Makes it easier to test. Signed-off-by: Noel Georgi <git@frezbo.dev>
siderolabs · Aug 17, 2023 · 6b0373e · 6b0373e
1 parent 52b3d8d
commit 6b0373e
Show file tree

Hide file tree

Showing 16 changed files with 834 additions and 180 deletions.
diff --git a/.drone.jsonnet b/.drone.jsonnet
@@ -443,7 +443,7 @@ local integration_qemu_trusted_boot = Step('e2e-qemu-trusted-boot', target='e2e-
   IMAGE_REGISTRY: local_registry,
   VIA_MAINTENANCE_MODE: 'true',
   WITH_TRUSTED_BOOT_ISO: 'true',
-  WITH_TEST: 'validate_booted_secureboot',
+  EXTRA_TEST_ARGS: '-talos.trustedboot',
 });
 
 local build_race = Step('build-race', target='initramfs installer', depends_on=[load_artifacts], environment={ IMAGE_REGISTRY: local_registry, PUSH: true, TAG_SUFFIX: '-race', WITH_RACE: '1', PLATFORM: 'linux/amd64' });
@@ -455,11 +455,10 @@ local integration_provision_tests_track_1 = Step('provision-tests-track-1', priv
 local integration_provision_tests_track_2 = Step('provision-tests-track-2', privileged=true, depends_on=[integration_provision_tests_prepare], environment={ IMAGE_REGISTRY: local_registry });
 
 local integration_extensions = Step('e2e-extensions', target='e2e-qemu', privileged=true, depends_on=[extensions_patch_manifest], environment={
-  SHORT_INTEGRATION_TEST: 'yes',
-  QEMU_MEMORY_WORKERS: '3072',
+  QEMU_MEMORY_WORKERS: '4096',
   WITH_CONFIG_PATCH_WORKER: '@_out/extensions-patch.json',
-  WITH_TEST: 'run_extensions_test',
   IMAGE_REGISTRY: local_registry,
+  EXTRA_TEST_ARGS: '-talos.extensions.testtype=qemu',
 });
 local integration_cilium = Step('e2e-cilium', target='e2e-qemu', privileged=true, depends_on=[load_artifacts], environment={
   SHORT_INTEGRATION_TEST: 'yes',

diff --git a/Makefile b/Makefile
@@ -16,7 +16,7 @@ CLOUD_IMAGES_EXTRA_ARGS ?= ""
 
 ARTIFACTS := _out
 TOOLS ?= ghcr.io/siderolabs/tools:v1.5.0
-PKGS ?= v1.5.0
+PKGS ?= v1.6.0-alpha.0-5-g7717b7e
 EXTRAS ?= v1.5.0
 # renovate: datasource=github-tags depName=golang/go
 GO_VERSION ?= 1.20

diff --git a/hack/test/e2e-qemu.sh b/hack/test/e2e-qemu.sh
@@ -195,8 +195,6 @@ case "${TEST_MODE:-default}" in
     ;;
   *)
     get_kubeconfig
-    validate_virtio_modules
-    validate_rlimit_nofile
     run_talos_integration_test
     run_kubernetes_integration_test
 

diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh
@@ -150,7 +150,7 @@ function run_talos_integration_test {
       ;;
   esac
 
-  "${INTEGRATION_TEST}" -test.v -talos.failfast -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}"
+  "${INTEGRATION_TEST}" -test.v -talos.failfast -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}"
 }
 
 function run_talos_integration_test_docker {
@@ -170,7 +170,7 @@ function run_talos_integration_test_docker {
       ;;
   esac
 
-  "${INTEGRATION_TEST}" -test.v -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.k8sendpoint 127.0.0.1:6443 -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}"
+  "${INTEGRATION_TEST}" -test.v -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.k8sendpoint 127.0.0.1:6443 -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}"
 }
 
 function run_kubernetes_conformance_test {
@@ -221,81 +221,6 @@ function build_registry_mirrors {
   fi
 }
 
-function run_extensions_test {
-  # e2e-qemu creates 3 controlplanes
-  # use a worker node to test extensions
-  "${TALOSCTL}" config node 172.20.1.5
-
-  echo "Testing firmware extensions..."
-  ${TALOSCTL} ls /lib/firmware | grep amd-ucode
-  ${TALOSCTL} ls /lib/firmware | grep bnx2x
-  ${TALOSCTL} ls /lib/firmware | grep i915
-  ${TALOSCTL} ls /lib/firmware | grep intel-ucode
-
-  echo "Testing kernel modules tree extension..."
-  ${TALOSCTL} get extensions modules.dep
-  KERNEL_VERSION=$(${TALOSCTL} get extensions modules.dep -o json | jq -r '.spec.metadata.version')
-  ${TALOSCTL} ls "/lib/modules/${KERNEL_VERSION}/extras/" | grep gasket
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/modules.dep" | grep -E gasket
-  ${TALOSCTL} ls "/lib/modules/${KERNEL_VERSION}/extras/" | grep drbd
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/modules.dep" | grep -E drbd
-  ${TALOSCTL} ls "/lib/modules/${KERNEL_VERSION}/kernel/drivers/video/" | grep nvidia
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/modules.dep" | grep -E nvidia
-
-  echo "Testing drbd and gasket modules are loaded..."
-  ${TALOSCTL} read /proc/modules | grep -E drbd
-  ${TALOSCTL} read /proc/modules | grep -E gasket
-
-  echo "Testing kernel modules signature..."
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/extras/drbd.ko" | ${MODULE_SIG_VERIFY} -cert "${KERNEL_MODULE_SIGNING_PUBLIC_KEY}" -module -
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/extras/gasket.ko" | ${MODULE_SIG_VERIFY} -cert "${KERNEL_MODULE_SIGNING_PUBLIC_KEY}" -module -
-  ${TALOSCTL} read "/lib/modules/${KERNEL_VERSION}/kernel/drivers/video/nvidia.ko" | ${MODULE_SIG_VERIFY} -cert "${KERNEL_MODULE_SIGNING_PUBLIC_KEY}" -module -
-
-  echo "Testing iscsi-tools extensions service..."
-  ${TALOSCTL} services ext-iscsid | grep -E "STATE\s+Running"
-  ${TALOSCTL} services ext-tgtd | grep -E "STATE\s+Running"
-
-  echo "Testing nut-client extensions service..."
-  ${TALOSCTL} services ext-nut-client | grep -E "STATE\s+Running"
-
-  echo "Testing gVsisor..."
-  ${KUBECTL} apply -f "${PWD}/hack/test/gvisor/manifest.yaml"
-  sleep 10
-  ${KUBECTL} wait --for=condition=ready pod nginx-gvisor --timeout=2m
-
-  echo "Testing hello-world extension service..."
-  ${TALOSCTL} services ext-hello-world | grep -E "STATE\s+Running"
-  curl http://172.20.1.5/ | grep Hello
-
-  echo "Testing tailscale extension service..."
-  ${TALOSCTL} services ext-tailscale | grep -E "STATE\s+Running"
-  ${TALOSCTL} get links tailscale0
-
-  echo "Testing qemu-guest-agent extension service..."
-  ${TALOSCTL} services ext-qemu-guest-agent | grep -E "STATE\s+Running"
-  # get exisitng boot id
-  BOOT_ID=$(get_boot_id)
-  NODE_HOSTNAME=$(${TALOSCTL} get hostname -o json | jq -r '.spec.hostname')
-  CLUSTERNAME=$(cut -d '-' -f 1-2 <<< "${NODE_HOSTNAME}")
-  # issue a reboot via qemu-guest-agent
-  echo '{"execute":"guest-shutdown", "arguments": {"mode": "reboot"}}' | socat - unix-connect:"${HOME}/.talos/clusters/${CLUSTERNAME}/${NODE_HOSTNAME}.sock"
-  # wait for the node to reboot
-  ${TALOSCTL} -n 172.20.1.2 health
-  NEW_BOOT_ID=$(get_boot_id)
-  # verify that the boot id has changed
-  if [ "${BOOT_ID}" == "${NEW_BOOT_ID}" ]; then
-    echo "ERROR: boot id has not changed, reboot failed"
-    exit 1
-  fi
-
-  # set talosctl config back to the first controlplane
-  "${TALOSCTL}" config node 172.20.1.2
-}
-
-function get_boot_id() {
-  ${TALOSCTL} read /proc/sys/kernel/random/boot_id
-}
-
 function run_csi_tests {
   ${HELM} repo add rook-release https://charts.rook.io/release
   ${HELM} repo update
@@ -314,21 +239,6 @@ function run_csi_tests {
   KUBERNETES_SERVICE_HOST="" KUBECONFIG="${TMP}/kubeconfig" "${KUBESTR}" fio --storageclass ceph-block --size 10G
 }
 
-function validate_virtio_modules {
-  ${TALOSCTL} read /proc/modules | grep -q virtio
-}
-
-function validate_rlimit_nofile {
-  # verify that RLIMIT_NOFILE is set to 1048576
-  ${KUBECTL} run --rm --restart=Never -it rlimit-test --image=alpine -- /bin/sh -c "ulimit -n" | grep 1048576
-}
-
-function validate_booted_secureboot {
-  ${TALOSCTL} dmesg | grep "Secure boot enabled"
-  ${TALOSCTL} get securitystate -o json
-  ${TALOSCTL} get securitystate -o json | jq -e '.spec.secureBoot == true'
-}
-
 function install_and_run_cilium_cni_tests {
   get_kubeconfig
 

diff --git a/hack/test/extensions/extension-patch-filter.jq b/hack/test/extensions/extension-patch-filter.jq
@@ -4,7 +4,7 @@
         "path": "/machine/install/extensions",
         "value": [
             {
-                "image": map(select(. | contains("nvidia-container-toolkit") or contains("nvidia-fabricmanager") | not)) | .[]
+                "image": map(select(. | contains("nvidia") or contains("tailscale") | not)) | .[]
             }
         ]
     },
@@ -20,11 +20,68 @@
         "path": "/machine/kernel",
         "value": {
             "modules": [
+                {
+                    "name": "asix"
+                },
+                {
+                    "name": "ax88179_178a"
+                },
+                {
+                    "name": "ax88796b"
+                },
+                {
+                    "name": "cdc_ether"
+                },
+                {
+                    "name": "cdc_mbim"
+                },
+                {
+                    "name": "cdc_ncm"
+                },
+                {
+                    "name": "cdc_subset"
+                },
+                {
+                    "name": "cdc_wdm"
+                },
                 {
                     "name": "drbd"
                 },
                 {
                     "name": "gasket"
+                },
+                {
+                    "name": "net1080"
+                },
+                {
+                    "name": "option"
+                },
+                {
+                    "name": "qmi_wwan"
+                },
+                {
+                    "name": "r8153_ecm"
+                },
+                {
+                    "name": "thunderbolt"
+                },
+                {
+                    "name": "thunderbolt_net"
+                },
+                {
+                    "name": "usb_wwan"
+                },
+                {
+                    "name": "usbnet"
+                },
+                {
+                    "name": "usbserial"
+                },
+                {
+                    "name": "zaurus"
+                },
+                {
+                    "name": "zfs"
                 }
             ]
         }

diff --git a/internal/integration/api/api.go b/internal/integration/api/api.go
@@ -11,6 +11,11 @@ import "github.com/stretchr/testify/suite"
 
 var allSuites []suite.TestingSuite
 
+const (
+	provisionerDocker = "docker"
+	provisionerQEMU   = "qemu"
+)
+
 // GetAllSuites returns all the suites for API test.
 //
 // Depending on build tags, this might return different lists.