From 1f73170a8007ac75035c041540d9bbaea4e260cd Mon Sep 17 00:00:00 2001 From: Nir Soffer Date: Sun, 25 Aug 2024 08:37:51 +0300 Subject: [PATCH] Add trivial e2e test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e module contains a helper tool (e2e) and a test suite (gather_test.go). The helper tool creates and deletes 2 kind clusters. This takes about 35 seconds, and will likely take more when the cluster are more interesting, so we keep it outside of the test suite. This way we can run tests very quickly with the same cluster. Since the gathering from a cluster is mostly immutable, reusing the same cluster is safe. Issues: - The test suite contain one trivial test, gathering both clusters - Gathered data is not validated yet - The logs are too noisy. We need to log debug logs only to test log. Normal output should include only the major events. - Test configurations and gathered output are stored in the e2e directory. This should move to a temporary test directory. Example test flow: $ make go build -o e2e ./cmd $ ./e2e Manage the e2e testing environment Usage: e2e [command] Available Commands: completion Generate the autocompletion script for the specified shell create Create the e2e environment delete Delete the e2e environment help Help about any command Flags: -h, --help help for e2e Use "e2e [command] --help" for more information about a command. $ ./e2e create 2024/08/25 09:53:11 Creating clusters 2024/08/25 09:53:11 Creating cluster "kind-c2" 2024/08/25 09:53:11 Creating cluster "kind-c1" 2024/08/25 09:53:11 Running /usr/local/bin/kind create cluster --name c2 --kubeconfig kind-c2.yaml --wait 60s 2024/08/25 09:53:11 Running /usr/local/bin/kind create cluster --name c1 --kubeconfig kind-c1.yaml --wait 60s 2024/08/25 09:53:11 Creating cluster "c2" ... 2024/08/25 09:53:11 â€ĸ Ensuring node image (kindest/node:v1.31.0) đŸ–ŧ ... 2024/08/25 09:53:11 Creating cluster "c1" ... 2024/08/25 09:53:11 â€ĸ Ensuring node image (kindest/node:v1.31.0) đŸ–ŧ ... 2024/08/25 09:53:11 ✓ Ensuring node image (kindest/node:v1.31.0) đŸ–ŧ 2024/08/25 09:53:11 â€ĸ Preparing nodes đŸ“Ļ ... 2024/08/25 09:53:11 ✓ Ensuring node image (kindest/node:v1.31.0) đŸ–ŧ 2024/08/25 09:53:11 â€ĸ Preparing nodes đŸ“Ļ ... 2024/08/25 09:53:12 ✓ Preparing nodes đŸ“Ļ 2024/08/25 09:53:12 ✓ Preparing nodes đŸ“Ļ 2024/08/25 09:53:12 â€ĸ Writing configuration 📜 ... 2024/08/25 09:53:12 â€ĸ Writing configuration 📜 ... 2024/08/25 09:53:12 ✓ Writing configuration 📜 2024/08/25 09:53:12 â€ĸ Starting control-plane 🕹ī¸ ... 2024/08/25 09:53:12 ✓ Writing configuration 📜 2024/08/25 09:53:12 â€ĸ Starting control-plane 🕹ī¸ ... 2024/08/25 09:53:23 ✓ Starting control-plane 🕹ī¸ 2024/08/25 09:53:23 â€ĸ Installing CNI 🔌 ... 2024/08/25 09:53:23 ✓ Installing CNI 🔌 2024/08/25 09:53:23 â€ĸ Installing StorageClass 💾 ... 2024/08/25 09:53:23 ✓ Starting control-plane 🕹ī¸ 2024/08/25 09:53:23 â€ĸ Installing CNI 🔌 ... 2024/08/25 09:53:23 ✓ Installing StorageClass 💾 2024/08/25 09:53:23 â€ĸ Waiting ≤ 1m0s for control-plane = Ready âŗ ... 2024/08/25 09:53:23 ✓ Installing CNI 🔌 2024/08/25 09:53:23 â€ĸ Installing StorageClass 💾 ... 2024/08/25 09:53:24 ✓ Installing StorageClass 💾 2024/08/25 09:53:24 â€ĸ Waiting ≤ 1m0s for control-plane = Ready âŗ ... 2024/08/25 09:53:40 ✓ Waiting ≤ 1m0s for control-plane = Ready âŗ 2024/08/25 09:53:40 â€ĸ Ready after 17s 💚 2024/08/25 09:53:40 Set kubectl context to "kind-c1" 2024/08/25 09:53:40 You can now use your cluster with: 2024/08/25 09:53:40 2024/08/25 09:53:40 kubectl cluster-info --context kind-c1 --kubeconfig kind-c1.yaml 2024/08/25 09:53:40 2024/08/25 09:53:40 Thanks for using kind! 😊 2024/08/25 09:53:43 ✓ Waiting ≤ 1m0s for control-plane = Ready âŗ 2024/08/25 09:53:43 â€ĸ Ready after 19s 💚 2024/08/25 09:53:43 Set kubectl context to "kind-c2" 2024/08/25 09:53:43 You can now use your cluster with: 2024/08/25 09:53:43 2024/08/25 09:53:43 kubectl cluster-info --context kind-c2 --kubeconfig kind-c2.yaml 2024/08/25 09:53:43 2024/08/25 09:53:43 Thanks for using kind! 😊 2024/08/25 09:53:43 Merging cluster kubconfigs to "clusters.yaml" 2024/08/25 09:53:43 Running /usr/bin/kubectl config view --flatten 2024/08/25 09:53:43 Clusters created $ go test . -count=1 ok github.com/nirs/kubectl-gather/e2e 0.177s $ go test . -count=1 -v === RUN TestGather 2024/08/25 11:53:09 Running ../kubectl-gather --contexts kind-c1,kind-c2 --kubeconfig clusters.yaml --directory test-gather.out 2024/08/25 11:53:09 2024-08-25T11:53:09.144+0300 INFO gather Using kubeconfig "clusters.yaml" 2024/08/25 11:53:09 2024-08-25T11:53:09.144+0300 INFO gather Gathering from all namespaces 2024/08/25 11:53:09 2024-08-25T11:53:09.145+0300 INFO gather Using all addons 2024/08/25 11:53:09 2024-08-25T11:53:09.145+0300 INFO gather Gathering from cluster "kind-c1" 2024/08/25 11:53:09 2024-08-25T11:53:09.145+0300 INFO gather Gathering from cluster "kind-c2" 2024/08/25 11:53:09 2024-08-25T11:53:09.310+0300 INFO gather Gathered 320 resources from cluster "kind-c1" in 0.166 seconds 2024/08/25 11:53:09 2024-08-25T11:53:09.316+0300 INFO gather Gathered 338 resources from cluster "kind-c2" in 0.171 seconds 2024/08/25 11:53:09 2024-08-25T11:53:09.316+0300 INFO gather Gathered 658 resources from 2 clusters in 0.171 seconds --- PASS: TestGather (0.19s) PASS ok github.com/nirs/kubectl-gather/e2e 0.188s $ ./e2e delete 2024/08/25 09:57:43 Deleting clusters 2024/08/25 09:57:43 Deleting cluster "kind-c2" 2024/08/25 09:57:43 Deleting cluster "kind-c1" 2024/08/25 09:57:43 Running /usr/local/bin/kind delete cluster --name c2 --kubeconfig kind-c2.yaml 2024/08/25 09:57:43 Running /usr/local/bin/kind delete cluster --name c1 --kubeconfig kind-c1.yaml 2024/08/25 09:57:43 Deleting cluster "c2" ... 2024/08/25 09:57:43 Deleting cluster "c1" ... 2024/08/25 09:57:43 Deleted nodes: ["c2-control-plane"] 2024/08/25 09:57:43 Deleted nodes: ["c1-control-plane"] 2024/08/25 09:57:43 Clusters deleted --- .gitignore | 1 + e2e/Makefile | 2 + e2e/clusters/clusters.go | 134 +++++++++++++++++++++++++++++++++++++++ e2e/cmd/main.go | 46 ++++++++++++++ e2e/commands/commands.go | 38 +++++++++++ e2e/gather_test.go | 25 ++++++++ e2e/go.mod | 10 +++ e2e/go.sum | 10 +++ 8 files changed, 266 insertions(+) create mode 100644 e2e/Makefile create mode 100644 e2e/clusters/clusters.go create mode 100644 e2e/cmd/main.go create mode 100644 e2e/commands/commands.go create mode 100644 e2e/gather_test.go create mode 100644 e2e/go.mod create mode 100644 e2e/go.sum diff --git a/.gitignore b/.gitignore index 92a96f3..7a0c135 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /kubectl-gather /gather/ +/e2e/e2e diff --git a/e2e/Makefile b/e2e/Makefile new file mode 100644 index 0000000..0b4a841 --- /dev/null +++ b/e2e/Makefile @@ -0,0 +1,2 @@ +e2e: + go build -o $@ ./cmd diff --git a/e2e/clusters/clusters.go b/e2e/clusters/clusters.go new file mode 100644 index 0000000..c4daba0 --- /dev/null +++ b/e2e/clusters/clusters.go @@ -0,0 +1,134 @@ +package clusters + +import ( + "fmt" + "log" + "os" + "os/exec" + "slices" + "strings" + "sync" + + "github.com/nirs/kubectl-gather/e2e/commands" +) + +const kubeconfig = "clusters.yaml" + +var names = []string{"kind-c1", "kind-c2"} + +func Names() []string { + return names +} + +func Kubeconfig() string { + return kubeconfig +} + +func Create() error { + log.Print("Creating clusters") + if err := execute(createCluster, names); err != nil { + return err + } + if err := createKubeconfig(); err != nil { + return err + } + log.Print("Clusters created") + return nil +} + +func Delete() error { + log.Print("Deleting clusters") + if err := execute(deleteCluster, names); err != nil { + return err + } + _ = os.Remove(kubeconfig) + log.Print("Clusters deleted") + return nil +} + +func execute(fn func(name string) error, names []string) error { + errors := make(chan error, len(names)) + wg := sync.WaitGroup{} + for _, name := range names { + wg.Add(1) + go func() { + defer wg.Done() + err := fn(name) + if err != nil { + errors <- err + } + }() + } + wg.Wait() + close(errors) + for e := range errors { + return e + } + return nil +} + +func createCluster(name string) error { + log.Printf("Creating cluster %q", name) + exists, err := clusterExists(name) + if err != nil { + return err + } + if exists { + log.Printf("Using existing cluster: %q", name) + return nil + } + cmd := exec.Command( + "kind", "create", "cluster", + "--name", kindName(name), + "--kubeconfig", name+".yaml", + "--wait", "60s", + ) + return commands.LogStderr(cmd) +} + +func deleteCluster(name string) error { + log.Printf("Deleting cluster %q", name) + config := name + ".yaml" + cmd := exec.Command( + "kind", "delete", "cluster", + "--name", kindName(name), + "--kubeconfig", config, + ) + if err := commands.LogStderr(cmd); err != nil { + return err + } + _ = os.Remove(config) + return nil +} + +func createKubeconfig() error { + log.Printf("Creating kubconfigs %q", kubeconfig) + var configs []string + for _, name := range names { + configs = append(configs, name+".yaml") + } + cmd := exec.Command("kubectl", "config", "view", "--flatten") + cmd.Env = append(os.Environ(), "KUBECONFIG="+strings.Join(configs, ":")) + log.Printf("Running %v", cmd) + data, err := cmd.Output() + if err != nil { + return fmt.Errorf("Failed to merge configs: %s: %s", err, commands.Stderr(err)) + } + return os.WriteFile(kubeconfig, data, 0640) +} + +func clusterExists(name string) (bool, error) { + cmd := exec.Command("kind", "get", "clusters") + log.Printf("Running %v", cmd) + out, err := cmd.Output() + if err != nil { + return false, fmt.Errorf("Failed to get clusters: %s: %s", err, commands.Stderr(err)) + } + trimmed := strings.TrimSpace(string(out)) + existing := strings.Split(trimmed, "\n") + return slices.Contains(existing, kindName(name)), nil +} + +func kindName(name string) string { + return strings.TrimPrefix(name, "kind-") +} diff --git a/e2e/cmd/main.go b/e2e/cmd/main.go new file mode 100644 index 0000000..063c3fb --- /dev/null +++ b/e2e/cmd/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "log" + "os" + + "github.com/nirs/kubectl-gather/e2e/clusters" + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "e2e", + Short: "Manage the e2e testing environment", +} + +var createCmd = &cobra.Command{ + Use: "create", + Short: "Create the e2e environment", + Run: func(cmd *cobra.Command, args []string) { + if err := clusters.Create(); err != nil { + log.Fatal(err) + } + }, +} + +var deleteCmd = &cobra.Command{ + Use: "delete", + Short: "Delete the e2e environment", + Run: func(cmd *cobra.Command, args []string) { + if err := clusters.Delete(); err != nil { + log.Fatal(err) + } + }, +} + +func init() { + rootCmd.AddCommand(deleteCmd) + rootCmd.AddCommand(createCmd) +} + +func main() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} diff --git a/e2e/commands/commands.go b/e2e/commands/commands.go new file mode 100644 index 0000000..422095c --- /dev/null +++ b/e2e/commands/commands.go @@ -0,0 +1,38 @@ +package commands + +import ( + "bufio" + "io" + "log" + "os/exec" +) + +func LogStderr(cmd *exec.Cmd) error { + log.Printf("Running %v", cmd) + pipe, err := cmd.StderrPipe() + if err != nil { + return err + } + if err := cmd.Start(); err != nil { + return err + } + reader := bufio.NewReader(pipe) + for { + line, _, err := reader.ReadLine() + if err != nil { + if err != io.EOF { + log.Printf("Failed to read from command stderr: %s", err) + } + break + } + log.Print(string(line)) + } + return cmd.Wait() +} + +func Stderr(err error) []byte { + if ee, ok := err.(*exec.ExitError); ok { + return ee.Stderr + } + return nil +} diff --git a/e2e/gather_test.go b/e2e/gather_test.go new file mode 100644 index 0000000..1f7603c --- /dev/null +++ b/e2e/gather_test.go @@ -0,0 +1,25 @@ +package e2e_test + +import ( + "os/exec" + "strings" + "testing" + + "github.com/nirs/kubectl-gather/e2e/clusters" + "github.com/nirs/kubectl-gather/e2e/commands" +) + +const executable = "../kubectl-gather" + +func TestGather(t *testing.T) { + cmd := exec.Command( + executable, + "--contexts", strings.Join(clusters.Names(), ","), + "--kubeconfig", clusters.Kubeconfig(), + "--directory", "test-gather.out", + ) + if err := commands.LogStderr(cmd); err != nil { + t.Errorf("kubectl-gather failed: %s", err) + } + // XXX verify gathered data. +} diff --git a/e2e/go.mod b/e2e/go.mod new file mode 100644 index 0000000..1f15bf9 --- /dev/null +++ b/e2e/go.mod @@ -0,0 +1,10 @@ +module github.com/nirs/kubectl-gather/e2e + +go 1.23 + +require github.com/spf13/cobra v1.8.1 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/e2e/go.sum b/e2e/go.sum new file mode 100644 index 0000000..912390a --- /dev/null +++ b/e2e/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=