🤖 Add upgrade test

tests/assets/suc.yaml

@@ -0,0 +1,21 @@
+---
+apiVersion: upgrade.cattle.io/v1
+kind: Plan
+metadata:
+  name: os-upgrade
+  namespace: system-upgrade
+  labels:
+    k3s-upgrade: server
+spec:
+  concurrency: 1
+  #version:  latest
+  version: "opensuse-v1.23.5-44"
+  nodeSelector:
+    matchExpressions:
+      - {key: kubernetes.io/hostname, operator: Exists}
+  serviceAccountName: system-upgrade
+  cordon: false
+  upgrade:
+    image: quay.io/c3os/c3os
+    command:
+    - "/usr/sbin/suc-upgrade"

tests/machine/machine.go

@@ -178,16 +178,19 @@ func connectToHost() (*ssh.Client, *ssh.Session, error) {
 	return client, session, nil
 }
 
+func Sudo(c string) (string, error) {
+	return SSHCommand(fmt.Sprintf(`sudo /bin/sh -c "%s"`, c))
+}
+
 // GatherAllLogs will try to gather as much info from the system as possible, including services, dmesg and os related info
 func GatherAllLogs(services []string, logFiles []string) {
 	// services
 	for _, ser := range services {
-		out, err := SSHCommand(fmt.Sprintf("sudo journalctl -u %s -o short-iso >> /run/%s.log", ser, ser))
+		out, err := Sudo(fmt.Sprintf("journalctl -u %s -o short-iso >> /run/%s.log", ser, ser))
 		if err != nil {
 			fmt.Printf("Error getting journal for service %s: %s\n", ser, err.Error())
 			fmt.Printf("Output from command: %s\n", out)
 		}
-		SSHCommand(fmt.Sprintf("/run/%s.log", ser))
 		GatherLog(fmt.Sprintf("/run/%s.log", ser))
 	}
 
@@ -197,36 +200,36 @@ func GatherAllLogs(services []string, logFiles []string) {
 	}
 
 	// dmesg
-	out, err := SSHCommand("sudo dmesg > /run/dmesg")
+	out, err := Sudo("dmesg > /run/dmesg")
 	if err != nil {
 		fmt.Printf("Error getting dmesg : %s\n", err.Error())
 		fmt.Printf("Output from command: %s\n", out)
 	}
 	GatherLog("/run/dmesg")
 
 	// grab full journal
-	out, err = SSHCommand("sudo journalctl -o short-iso > /run/journal.log")
+	out, err = Sudo("journalctl -o short-iso > /run/journal.log")
 	if err != nil {
 		fmt.Printf("Error getting full journalctl info : %s\n", err.Error())
 		fmt.Printf("Output from command: %s\n", out)
 	}
 	GatherLog("/run/journal.log")
 
 	// uname
-	out, err = SSHCommand("uname -a > /run/uname.log")
+	out, err = Sudo("uname -a > /run/uname.log")
 	if err != nil {
 		fmt.Printf("Error getting uname info : %s\n", err.Error())
 		fmt.Printf("Output from command: %s\n", out)
 	}
 	GatherLog("/run/uname.log")
 
 	// disk info
-	out, err = SSHCommand("sudo lsblk -a >> /run/disks.log")
+	out, err = Sudo("lsblk -a >> /run/disks.log")
 	if err != nil {
 		fmt.Printf("Error getting disk info : %s\n", err.Error())
 		fmt.Printf("Output from command: %s\n", out)
 	}
-	out, err = SSHCommand("sudo blkid >> /run/disks.log")
+	out, err = Sudo("blkid >> /run/disks.log")
 	if err != nil {
 		fmt.Printf("Error getting disk info : %s\n", err.Error())
 		fmt.Printf("Output from command: %s\n", out)
@@ -241,7 +244,7 @@ func GatherAllLogs(services []string, logFiles []string) {
 
 // GatherLog will try to scp the given log from the machine to a local file
 func GatherLog(logPath string) {
-	SSHCommand("sudo chmod 777 " + logPath)
+	Sudo("chmod 777 " + logPath)
 	fmt.Printf("Trying to get file: %s\n", logPath)
 	sshConfig := &ssh.ClientConfig{
 		User:    user(),

tests/upgrade_test.go

@@ -1,7 +1,11 @@
 package mos_test
 
 import (
+	"bytes"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
 	"os"
 	"time"
 
@@ -17,11 +21,14 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 
 	AfterEach(func() {
 		if CurrentGinkgoTestDescription().Failed {
-			machine.SSHCommand("sudo k3s kubectl get pods -A -o json > /run/pods.json")
-			machine.SSHCommand("sudo k3s kubectl get events -A -o json > /run/events.json")
-			machine.SSHCommand("sudo df -h > /run/disk")
-			machine.SSHCommand("sudo mount > /run/mounts")
-			machine.SSHCommand("sudo blkid > /run/blkid")
+			machine.Sudo("k3s kubectl get pods -A -o json > /run/pods.json")
+			machine.Sudo("k3s kubectl get events -A -o json > /run/events.json")
+			machine.Sudo("cat /proc/cmdline > /run/cmdline")
+			machine.Sudo("chmod 777 /run/events.json")
+
+			machine.Sudo("df -h > /run/disk")
+			machine.Sudo("mount > /run/mounts")
+			machine.Sudo("blkid > /run/blkid")
 
 			machine.GatherAllLogs(
 				[]string{
@@ -40,14 +47,15 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 					"/run/mounts",
 					"/run/blkid",
 					"/run/events.json",
+					"/run/cmdline",
 				})
 		}
 	})
 
 	Context("live cd", func() {
 		It("has default service active", func() {
 			if os.Getenv("FLAVOR") == "alpine" {
-				out, _ := machine.SSHCommand("sudo rc-status")
+				out, _ := machine.Sudo("rc-status")
 				Expect(out).Should(ContainSubstring("c3os"))
 				Expect(out).Should(ContainSubstring("c3os-agent"))
 			} else {
@@ -56,7 +64,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 				// 	return out
 				// }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token"))
 
-				out, _ := machine.SSHCommand("sudo systemctl status c3os")
+				out, _ := machine.Sudo("systemctl status c3os")
 				Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os.service; enabled; vendor preset: disabled)"))
 			}
 		})
@@ -67,10 +75,10 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 			err := machine.SendFile("assets/single.yaml", "/tmp/config.yaml", "0770")
 			Expect(err).ToNot(HaveOccurred())
 
-			out, _ := machine.SSHCommand("sudo elemental install --cloud-init /tmp/config.yaml /dev/sda")
+			out, _ := machine.Sudo("elemental install --cloud-init /tmp/config.yaml /dev/sda")
 			Expect(out).Should(ContainSubstring("COS_ACTIVE"))
 			fmt.Println(out)
-			machine.SSHCommand("sudo sync")
+			machine.Sudo("sync")
 			machine.DetachCD()
 			machine.Restart()
 		})
@@ -80,7 +88,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 
 		It("has default services on", func() {
 			if os.Getenv("FLAVOR") == "alpine" {
-				out, _ := machine.SSHCommand("sudo rc-status")
+				out, _ := machine.Sudo("rc-status")
 				Expect(out).Should(ContainSubstring("c3os"))
 				Expect(out).Should(ContainSubstring("c3os-agent"))
 			} else {
@@ -89,10 +97,10 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 				// 	return out
 				// }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token"))
 
-				out, _ := machine.SSHCommand("sudo systemctl status c3os-agent")
+				out, _ := machine.Sudo("systemctl status c3os-agent")
 				Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os-agent.service; enabled; vendor preset: disabled)"))
 
-				out, _ = machine.SSHCommand("sudo systemctl status systemd-timesyncd")
+				out, _ = machine.Sudo("systemctl status systemd-timesyncd")
 				Expect(out).Should(ContainSubstring("loaded (/usr/lib/systemd/system/systemd-timesyncd.service; enabled; vendor preset: disabled)"))
 			}
 		})
@@ -101,17 +109,68 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() {
 			Eventually(func() string {
 				var out string
 				if os.Getenv("FLAVOR") == "alpine" {
-					out, _ = machine.SSHCommand("sudo systemctl status c3os-agent")
+					out, _ = machine.Sudo("systemctl status c3os-agent")
 				} else {
-					out, _ = machine.SSHCommand("sudo systemctl status c3os-agent")
+					out, _ = machine.Sudo("systemctl status c3os-agent")
 				}
 				return out
 			}, 900*time.Second, 10*time.Second).Should(ContainSubstring("One time bootstrap starting"))
 
 			Eventually(func() string {
-				out, _ := machine.SSHCommand("sudo cat /etc/rancher/k3s/k3s.yaml")
+				out, _ := machine.Sudo("cat /etc/rancher/k3s/k3s.yaml")
 				return out
 			}, 900*time.Second, 10*time.Second).Should(ContainSubstring("https:"))
+
+			By("installing system-upgrade-controller", func() {
+
+				kubectl := func(s string) (string, error) {
+					return machine.Sudo("k3s kubectl " + s)
+				}
+
+				resp, err := http.Get("https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml")
+				Expect(err).ToNot(HaveOccurred())
+				defer resp.Body.Close()
+				data := bytes.NewBuffer([]byte{})
+
+				_, err = io.Copy(data, resp.Body)
+				Expect(err).ToNot(HaveOccurred())
+
+				temp, err := ioutil.TempFile("", "temp")
+				Expect(err).ToNot(HaveOccurred())
+
+				defer os.RemoveAll(temp.Name())
+				err = ioutil.WriteFile(temp.Name(), data.Bytes(), os.ModePerm)
+				Expect(err).ToNot(HaveOccurred())
+
+				err = machine.SendFile(temp.Name(), "/tmp/kubectl.yaml", "0770")
+				Expect(err).ToNot(HaveOccurred())
+
+				Eventually(func() string {
+					out, _ := kubectl("apply -f /tmp/kubectl.yaml")
+					return out
+				}, 900*time.Second, 10*time.Second).Should(ContainSubstring("unchanged"))
+
+				err = machine.SendFile("assets/suc.yaml", "/tmp/suc.yaml", "0770")
+				Expect(err).ToNot(HaveOccurred())
+
+				Eventually(func() string {
+					out, _ := kubectl("apply -f /tmp/suc.yaml")
+					return out
+				}, 900*time.Second, 10*time.Second).Should(ContainSubstring("unchanged"))
+
+				Eventually(func() string {
+					out, _ := kubectl("get pods -A")
+					fmt.Println(out)
+					return out
+				}, 900*time.Second, 10*time.Second).Should(ContainSubstring("apply-os-upgrade-on-"))
+
+				Eventually(func() string {
+					out, _ := kubectl("get pods -A")
+					fmt.Println(out)
+					version, _ := machine.SSHCommand("source /etc/os-release; echo $VERSION")
+					return version
+				}, 20*time.Minute, 10*time.Second).Should(ContainSubstring("c3OS44"))
+			})
 		})
 
 	})