From 7a6ae06fd791afa23cd9356b2cfd34ca1413c300 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 8 May 2022 14:11:33 +0200 Subject: [PATCH] :robot: Add upgrade test --- tests/assets/suc.yaml | 21 ++++++++++ tests/machine/machine.go | 19 +++++---- tests/upgrade_test.go | 89 +++++++++++++++++++++++++++++++++------- 3 files changed, 106 insertions(+), 23 deletions(-) create mode 100644 tests/assets/suc.yaml diff --git a/tests/assets/suc.yaml b/tests/assets/suc.yaml new file mode 100644 index 000000000..bc52b7954 --- /dev/null +++ b/tests/assets/suc.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: upgrade.cattle.io/v1 +kind: Plan +metadata: + name: os-upgrade + namespace: system-upgrade + labels: + k3s-upgrade: server +spec: + concurrency: 1 + #version: latest + version: "opensuse-v1.23.5-44" + nodeSelector: + matchExpressions: + - {key: kubernetes.io/hostname, operator: Exists} + serviceAccountName: system-upgrade + cordon: false + upgrade: + image: quay.io/c3os/c3os + command: + - "/usr/sbin/suc-upgrade" diff --git a/tests/machine/machine.go b/tests/machine/machine.go index 64496f812..dc6d78705 100644 --- a/tests/machine/machine.go +++ b/tests/machine/machine.go @@ -178,16 +178,19 @@ func connectToHost() (*ssh.Client, *ssh.Session, error) { return client, session, nil } +func Sudo(c string) (string, error) { + return SSHCommand(fmt.Sprintf(`sudo /bin/sh -c "%s"`, c)) +} + // GatherAllLogs will try to gather as much info from the system as possible, including services, dmesg and os related info func GatherAllLogs(services []string, logFiles []string) { // services for _, ser := range services { - out, err := SSHCommand(fmt.Sprintf("sudo journalctl -u %s -o short-iso >> /run/%s.log", ser, ser)) + out, err := Sudo(fmt.Sprintf("journalctl -u %s -o short-iso >> /run/%s.log", ser, ser)) if err != nil { fmt.Printf("Error getting journal for service %s: %s\n", ser, err.Error()) fmt.Printf("Output from command: %s\n", out) } - SSHCommand(fmt.Sprintf("/run/%s.log", ser)) GatherLog(fmt.Sprintf("/run/%s.log", ser)) } @@ -197,7 +200,7 @@ func GatherAllLogs(services []string, logFiles []string) { } // dmesg - out, err := SSHCommand("sudo dmesg > /run/dmesg") + out, err := Sudo("dmesg > /run/dmesg") if err != nil { fmt.Printf("Error getting dmesg : %s\n", err.Error()) fmt.Printf("Output from command: %s\n", out) @@ -205,7 +208,7 @@ func GatherAllLogs(services []string, logFiles []string) { GatherLog("/run/dmesg") // grab full journal - out, err = SSHCommand("sudo journalctl -o short-iso > /run/journal.log") + out, err = Sudo("journalctl -o short-iso > /run/journal.log") if err != nil { fmt.Printf("Error getting full journalctl info : %s\n", err.Error()) fmt.Printf("Output from command: %s\n", out) @@ -213,7 +216,7 @@ func GatherAllLogs(services []string, logFiles []string) { GatherLog("/run/journal.log") // uname - out, err = SSHCommand("uname -a > /run/uname.log") + out, err = Sudo("uname -a > /run/uname.log") if err != nil { fmt.Printf("Error getting uname info : %s\n", err.Error()) fmt.Printf("Output from command: %s\n", out) @@ -221,12 +224,12 @@ func GatherAllLogs(services []string, logFiles []string) { GatherLog("/run/uname.log") // disk info - out, err = SSHCommand("sudo lsblk -a >> /run/disks.log") + out, err = Sudo("lsblk -a >> /run/disks.log") if err != nil { fmt.Printf("Error getting disk info : %s\n", err.Error()) fmt.Printf("Output from command: %s\n", out) } - out, err = SSHCommand("sudo blkid >> /run/disks.log") + out, err = Sudo("blkid >> /run/disks.log") if err != nil { fmt.Printf("Error getting disk info : %s\n", err.Error()) fmt.Printf("Output from command: %s\n", out) @@ -241,7 +244,7 @@ func GatherAllLogs(services []string, logFiles []string) { // GatherLog will try to scp the given log from the machine to a local file func GatherLog(logPath string) { - SSHCommand("sudo chmod 777 " + logPath) + Sudo("chmod 777 " + logPath) fmt.Printf("Trying to get file: %s\n", logPath) sshConfig := &ssh.ClientConfig{ User: user(), diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index c59e9a59c..8272c7965 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -1,7 +1,11 @@ package mos_test import ( + "bytes" "fmt" + "io" + "io/ioutil" + "net/http" "os" "time" @@ -17,11 +21,14 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { AfterEach(func() { if CurrentGinkgoTestDescription().Failed { - machine.SSHCommand("sudo k3s kubectl get pods -A -o json > /run/pods.json") - machine.SSHCommand("sudo k3s kubectl get events -A -o json > /run/events.json") - machine.SSHCommand("sudo df -h > /run/disk") - machine.SSHCommand("sudo mount > /run/mounts") - machine.SSHCommand("sudo blkid > /run/blkid") + machine.Sudo("k3s kubectl get pods -A -o json > /run/pods.json") + machine.Sudo("k3s kubectl get events -A -o json > /run/events.json") + machine.Sudo("cat /proc/cmdline > /run/cmdline") + machine.Sudo("chmod 777 /run/events.json") + + machine.Sudo("df -h > /run/disk") + machine.Sudo("mount > /run/mounts") + machine.Sudo("blkid > /run/blkid") machine.GatherAllLogs( []string{ @@ -40,6 +47,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { "/run/mounts", "/run/blkid", "/run/events.json", + "/run/cmdline", }) } }) @@ -47,7 +55,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { Context("live cd", func() { It("has default service active", func() { if os.Getenv("FLAVOR") == "alpine" { - out, _ := machine.SSHCommand("sudo rc-status") + out, _ := machine.Sudo("rc-status") Expect(out).Should(ContainSubstring("c3os")) Expect(out).Should(ContainSubstring("c3os-agent")) } else { @@ -56,7 +64,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { // return out // }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token")) - out, _ := machine.SSHCommand("sudo systemctl status c3os") + out, _ := machine.Sudo("systemctl status c3os") Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os.service; enabled; vendor preset: disabled)")) } }) @@ -67,10 +75,10 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { err := machine.SendFile("assets/single.yaml", "/tmp/config.yaml", "0770") Expect(err).ToNot(HaveOccurred()) - out, _ := machine.SSHCommand("sudo elemental install --cloud-init /tmp/config.yaml /dev/sda") + out, _ := machine.Sudo("elemental install --cloud-init /tmp/config.yaml /dev/sda") Expect(out).Should(ContainSubstring("COS_ACTIVE")) fmt.Println(out) - machine.SSHCommand("sudo sync") + machine.Sudo("sync") machine.DetachCD() machine.Restart() }) @@ -80,7 +88,7 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { It("has default services on", func() { if os.Getenv("FLAVOR") == "alpine" { - out, _ := machine.SSHCommand("sudo rc-status") + out, _ := machine.Sudo("rc-status") Expect(out).Should(ContainSubstring("c3os")) Expect(out).Should(ContainSubstring("c3os-agent")) } else { @@ -89,10 +97,10 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { // return out // }, 30*time.Second, 10*time.Second).Should(ContainSubstring("no network token")) - out, _ := machine.SSHCommand("sudo systemctl status c3os-agent") + out, _ := machine.Sudo("systemctl status c3os-agent") Expect(out).Should(ContainSubstring("loaded (/etc/systemd/system/c3os-agent.service; enabled; vendor preset: disabled)")) - out, _ = machine.SSHCommand("sudo systemctl status systemd-timesyncd") + out, _ = machine.Sudo("systemctl status systemd-timesyncd") Expect(out).Should(ContainSubstring("loaded (/usr/lib/systemd/system/systemd-timesyncd.service; enabled; vendor preset: disabled)")) } }) @@ -101,17 +109,68 @@ var _ = Describe("k3s upgrade test", Label("upgrade"), func() { Eventually(func() string { var out string if os.Getenv("FLAVOR") == "alpine" { - out, _ = machine.SSHCommand("sudo systemctl status c3os-agent") + out, _ = machine.Sudo("systemctl status c3os-agent") } else { - out, _ = machine.SSHCommand("sudo systemctl status c3os-agent") + out, _ = machine.Sudo("systemctl status c3os-agent") } return out }, 900*time.Second, 10*time.Second).Should(ContainSubstring("One time bootstrap starting")) Eventually(func() string { - out, _ := machine.SSHCommand("sudo cat /etc/rancher/k3s/k3s.yaml") + out, _ := machine.Sudo("cat /etc/rancher/k3s/k3s.yaml") return out }, 900*time.Second, 10*time.Second).Should(ContainSubstring("https:")) + + By("installing system-upgrade-controller", func() { + + kubectl := func(s string) (string, error) { + return machine.Sudo("k3s kubectl " + s) + } + + resp, err := http.Get("https://github.com/rancher/system-upgrade-controller/releases/download/v0.9.1/system-upgrade-controller.yaml") + Expect(err).ToNot(HaveOccurred()) + defer resp.Body.Close() + data := bytes.NewBuffer([]byte{}) + + _, err = io.Copy(data, resp.Body) + Expect(err).ToNot(HaveOccurred()) + + temp, err := ioutil.TempFile("", "temp") + Expect(err).ToNot(HaveOccurred()) + + defer os.RemoveAll(temp.Name()) + err = ioutil.WriteFile(temp.Name(), data.Bytes(), os.ModePerm) + Expect(err).ToNot(HaveOccurred()) + + err = machine.SendFile(temp.Name(), "/tmp/kubectl.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() string { + out, _ := kubectl("apply -f /tmp/kubectl.yaml") + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("unchanged")) + + err = machine.SendFile("assets/suc.yaml", "/tmp/suc.yaml", "0770") + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() string { + out, _ := kubectl("apply -f /tmp/suc.yaml") + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("unchanged")) + + Eventually(func() string { + out, _ := kubectl("get pods -A") + fmt.Println(out) + return out + }, 900*time.Second, 10*time.Second).Should(ContainSubstring("apply-os-upgrade-on-")) + + Eventually(func() string { + out, _ := kubectl("get pods -A") + fmt.Println(out) + version, _ := machine.SSHCommand("source /etc/os-release; echo $VERSION") + return version + }, 20*time.Minute, 10*time.Second).Should(ContainSubstring("c3OS44")) + }) }) })