From 29347365eb90ef5be95061159649148ab8b7d2cb Mon Sep 17 00:00:00 2001 From: Daniel Pacak Date: Mon, 27 Jul 2020 16:59:05 +0200 Subject: [PATCH] test(kube-hunter): Add integration test for kube-hunter command Signed-off-by: Daniel Pacak --- .github/workflows/build.yaml | 9 ++ Makefile | 2 +- .../starboard_integration_test.go | 66 ---------- itest/integration_test.go | 116 ++++++++++++++++++ .../suite_test.go | 14 ++- pkg/cmd/kube_bench.go | 40 +++--- pkg/kubebench/scanner.go | 16 +-- pkg/kubehunter/crd/writer.go | 23 ++-- 8 files changed, 180 insertions(+), 106 deletions(-) delete mode 100644 integration-tests/starboard_integration_test.go create mode 100644 itest/integration_test.go rename integration-tests/starboard_integration_suite_test.go => itest/suite_test.go (72%) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 0b699302a..841d871c9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -42,15 +42,24 @@ jobs: file: ./coverage.txt - name: Setup Kubernetes cluster (KIND) uses: engineerd/setup-kind@v0.4.0 + with: + version: v0.8.1 - name: Test connection to Kubernetes cluster run: | kubectl cluster-info + kubectl describe node - name: Run integration tests run: | make integration-tests kubectl get crd env: KUBECONFIG: /home/runner/.kube/config + - name: Debug + if: always() + run: | + kubectl get events -n starboard + kubectl describe pod -n starboard + kubectl logs -n starboard job/$(kubectl get job -n starboard -o jsonpath='{.items[0].metadata.name}') - name: Release snapshot uses: goreleaser/goreleaser-action@v2 with: diff --git a/Makefile b/Makefile index 76cb4774a..2771fd5e1 100644 --- a/Makefile +++ b/Makefile @@ -10,4 +10,4 @@ unit-tests: $(SOURCES) go test -v -short -race -timeout 30s -coverprofile=coverage.txt -covermode=atomic ./... integration-tests: build - go test -v ./integration-tests + go test ./itest -ginkgo.v -ginkgo.progress -test.v diff --git a/integration-tests/starboard_integration_test.go b/integration-tests/starboard_integration_test.go deleted file mode 100644 index 21d8805f8..000000000 --- a/integration-tests/starboard_integration_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package integration_tests - -import ( - "context" - "os/exec" - - apiextentions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Starboard CLI", func() { - - BeforeEach(func() { - // currently do nothing - }) - - Describe("Running version command", func() { - It("should print the current version of the executable binary", func() { - cmd := exec.Command(pathToStarboardCLI, []string{"version"}...) - output, err := cmd.Output() - Expect(err).ToNot(HaveOccurred()) - Expect(string(output)).To(Equal("Starboard Version: {Version:dev Commit:none Date:unknown}\n")) - }) - - }) - - Describe("Running init command", func() { - It("should initialize Starboard", func() { - cmd := exec.Command(pathToStarboardCLI, []string{"init", "-v", "3"}...) - err := cmd.Run() - Expect(err).ToNot(HaveOccurred()) - - crdsList, err := apiextensionsClientset.CustomResourceDefinitions().List(context.TODO(), meta.ListOptions{}) - Expect(err).ToNot(HaveOccurred()) - - GetNames := func(crds []apiextentions.CustomResourceDefinition) []string { - names := make([]string, len(crds)) - for i, crd := range crds { - names[i] = crd.Name - } - return names - } - - Expect(crdsList.Items).To(WithTransform(GetNames, ContainElements( - "ciskubebenchreports.aquasecurity.github.io", - "configauditreports.aquasecurity.github.io", - "kubehunterreports.aquasecurity.github.io", - "vulnerabilities.aquasecurity.github.io", - ))) - - _, err = kubernetesClientset.CoreV1().Namespaces().Get(context.TODO(), "starboard", meta.GetOptions{}) - Expect(err).ToNot(HaveOccurred()) - - // TODO Assert other Kubernetes resources that we create in the init command - }) - }) - - AfterEach(func() { - // currently do nothing - }) - -}) diff --git a/itest/integration_test.go b/itest/integration_test.go new file mode 100644 index 000000000..52bb46cff --- /dev/null +++ b/itest/integration_test.go @@ -0,0 +1,116 @@ +package itest + +import ( + "context" + "fmt" + "os/exec" + "time" + + . "github.com/onsi/gomega/gbytes" + + "github.com/aquasecurity/starboard/pkg/kube" + . "github.com/onsi/gomega/gstruct" + + . "github.com/onsi/gomega/gexec" + apiextentions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Starboard CLI", func() { + + BeforeEach(func() { + command := exec.Command(pathToStarboardCLI, []string{"init", "-v", "3"}...) + session, err := Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + Eventually(session).Should(Exit(0)) + }) + + PDescribe("Running init command", func() { + It("should initialize Starboard", func() { + + crdsList, err := apiextensionsClientset.CustomResourceDefinitions().List(context.TODO(), meta.ListOptions{}) + Expect(err).ToNot(HaveOccurred()) + + GetNames := func(crds []apiextentions.CustomResourceDefinition) []string { + names := make([]string, len(crds)) + for i, crd := range crds { + names[i] = crd.Name + } + return names + } + + Expect(crdsList.Items).To(WithTransform(GetNames, ContainElements( + "ciskubebenchreports.aquasecurity.github.io", + "configauditreports.aquasecurity.github.io", + "kubehunterreports.aquasecurity.github.io", + "vulnerabilities.aquasecurity.github.io", + ))) + + _, err = kubernetesClientset.CoreV1().Namespaces().Get(context.TODO(), "starboard", meta.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + + // TODO Assert other Kubernetes resources that we create in the init command + }) + }) + + PDescribe("Running version command", func() { + It("should print the current version of the executable binary", func() { + command := exec.Command(pathToStarboardCLI, []string{"version"}...) + session, err := Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + Eventually(session).Should(Say("Starboard Version: {Version:dev Commit:none Date:unknown}\n")) + }) + }) + + Describe("Running kube-bench", func() { + It("should run kube-bench", func() { + command := exec.Command(pathToStarboardCLI, "kube-bench", "-v", "3", "--delete-scan-job=false") + session, err := Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + Eventually(session, 2*time.Minute).Should(Exit(0)) + + list, err := starboardClientset.AquasecurityV1alpha1().CISKubeBenchReports().List(context.TODO(), meta.ListOptions{}) + Expect(err).ToNot(HaveOccurred()) + _, _ = fmt.Fprintf(GinkgoWriter, "kube bench reports: %v\n", len(list.Items)) + }) + }) + + PDescribe("Running kube-hunter", func() { + It("should run kube-hunter", func() { + + command := exec.Command(pathToStarboardCLI, "kube-hunter", "-v", "3", "--delete-scan-job=false") + session, err := Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).ToNot(HaveOccurred()) + Eventually(session, 2*time.Minute).Should(Exit(0)) + + report, err := starboardClientset.AquasecurityV1alpha1().KubeHunterReports(). + Get(context.TODO(), "cluster", meta.GetOptions{}) + + Expect(err).ToNot(HaveOccurred()) + Expect(report.Labels).To(MatchAllKeys(Keys{ + kube.LabelResourceKind: Equal("Cluster"), + kube.LabelResourceName: Equal("cluster"), + })) + }) + }) + + //AfterEach(func() { + // command := exec.Command(pathToStarboardCLI, []string{"cleanup", "-v", "3"}...) + // session, err := Start(command, GinkgoWriter, GinkgoWriter) + // Expect(err).ToNot(HaveOccurred()) + // Eventually(session).Should(Exit(0)) + // + // // TODO We have to wait for the termination of the starboard namespace. Otherwise the init command fails + // // TODO when it attempts to create Kubernetes objects in the namespace that is being terminated. + // // TODO Maybe the cleanup command should block and wait unit the namespace is terminated? + // Eventually(func() bool { + // _, err := kubernetesClientset.CoreV1().Namespaces().Get(context.TODO(), "starboard", meta.GetOptions{}) + // return errors.IsNotFound(err) + // }, 10*time.Second).Should(BeTrue()) + //}) + +}) diff --git a/integration-tests/starboard_integration_suite_test.go b/itest/suite_test.go similarity index 72% rename from integration-tests/starboard_integration_suite_test.go rename to itest/suite_test.go index 4eef8bc9b..3c49f616f 100644 --- a/integration-tests/starboard_integration_suite_test.go +++ b/itest/suite_test.go @@ -1,10 +1,12 @@ -package integration_tests +package itest import ( "os" "testing" - "github.com/onsi/gomega/gexec" + starboardapi "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned" + + . "github.com/onsi/gomega/gexec" "k8s.io/client-go/tools/clientcmd" . "github.com/onsi/ginkgo" @@ -16,6 +18,7 @@ import ( var ( kubernetesClientset kubernetes.Interface apiextensionsClientset apiextensions.ApiextensionsV1beta1Interface + starboardClientset starboardapi.Interface ) var ( @@ -24,7 +27,7 @@ var ( var _ = BeforeSuite(func() { var err error - pathToStarboardCLI, err = gexec.Build("github.com/aquasecurity/starboard/cmd/starboard") + pathToStarboardCLI, err = Build("github.com/aquasecurity/starboard/cmd/starboard") Expect(err).ToNot(HaveOccurred()) config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) @@ -35,6 +38,9 @@ var _ = BeforeSuite(func() { apiextensionsClientset, err = apiextensions.NewForConfig(config) Expect(err).ToNot(HaveOccurred()) + + starboardClientset, err = starboardapi.NewForConfig(config) + Expect(err).ToNot(HaveOccurred()) }) func TestStarboardCLI(t *testing.T) { @@ -46,5 +52,5 @@ func TestStarboardCLI(t *testing.T) { } var _ = AfterSuite(func() { - gexec.CleanupBuildArtifacts() + CleanupBuildArtifacts() }) diff --git a/pkg/cmd/kube_bench.go b/pkg/cmd/kube_bench.go index f4d4af1db..b4f49f972 100644 --- a/pkg/cmd/kube_bench.go +++ b/pkg/cmd/kube_bench.go @@ -5,6 +5,8 @@ import ( "fmt" "sync" + core "k8s.io/api/core/v1" + "github.com/aquasecurity/starboard/pkg/ext" starboard "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned" "github.com/aquasecurity/starboard/pkg/kubebench" @@ -16,10 +18,6 @@ import ( "k8s.io/klog" ) -const ( - masterNodeLabel = "node-role.kubernetes.io/master" -) - func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command { cmd := &cobra.Command{ Use: "kube-bench", @@ -42,33 +40,35 @@ func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command { if err != nil { return } - // List Nodes nodeList, err := kubernetesClientset.CoreV1().Nodes().List(ctx, meta.ListOptions{}) if err != nil { - err = fmt.Errorf("list nodes: %w", err) + err = fmt.Errorf("listing nodes: %w", err) return } + scanner := kubebench.NewScanner(opts, kubernetesClientset) + writer := crd.NewWriter(ext.NewSystemClock(), starboardClientset) + var wg sync.WaitGroup - wg.Add(len(nodeList.Items)) - for _, nodeItem := range nodeList.Items { - target := "node" - if _, ok := nodeItem.Labels[masterNodeLabel]; ok { - target = "master" - } - nodeName := nodeItem.Name - go func() { - klog.V(3).Infof("Node name: %s Label:%s", nodeName, target) - report, node, err := kubebench.NewScanner(opts, kubernetesClientset).Scan(ctx, nodeName, target, &wg) + + for _, node := range nodeList.Items { + wg.Add(1) + go func(node core.Node) { + defer wg.Done() + + report, err := scanner.Scan(ctx, node) if err != nil { - klog.Warningf("Node name: %s Error NewScanner: %s", nodeName, err) + klog.Errorf("Error while running kube-bench on node: %s: %v", node.Name, err) + return } - err = crd.NewWriter(ext.NewSystemClock(), starboardClientset).Write(ctx, report, node) + err = writer.Write(ctx, report, &node) if err != nil { - klog.Warningf("Node name: %s Error NewWriter: %s", nodeName, err) + klog.Errorf("Error while writing kube-bench report for node: %s: %v", node.Name, err) + return } - }() + }(node) } + wg.Wait() return }, diff --git a/pkg/kubebench/scanner.go b/pkg/kubebench/scanner.go index 42fa9dc70..dc39a8381 100644 --- a/pkg/kubebench/scanner.go +++ b/pkg/kubebench/scanner.go @@ -3,7 +3,6 @@ package kubebench import ( "context" "fmt" - "sync" "github.com/aquasecurity/starboard/pkg/scanners" @@ -51,10 +50,9 @@ func NewScanner(opts kube.ScannerOpts, clientset kubernetes.Interface) *Scanner } } -func (s *Scanner) Scan(ctx context.Context, nodeName string, target string, wg *sync.WaitGroup) (report starboard.CISKubeBenchOutput, node *core.Node, err error) { - defer wg.Done() +func (s *Scanner) Scan(ctx context.Context, nodeItem core.Node) (report starboard.CISKubeBenchOutput, err error) { // 1. Prepare descriptor for the Kubernetes Job which will run kube-bench - job := s.prepareKubeBenchJob(nodeName, target) + job := s.prepareKubeBenchJob(nodeItem) // 2. Run the prepared Job and wait for its completion or failure err = runner.New().Run(ctx, kube.NewRunnableJob(s.clientset, job)) @@ -102,12 +100,14 @@ func (s *Scanner) Scan(ctx context.Context, nodeName string, target string, wg * return } - node, err = s.clientset.CoreV1().Nodes().Get(ctx, kubeBenchPod.Spec.NodeName, meta.GetOptions{}) - return } -func (s *Scanner) prepareKubeBenchJob(nodeName string, target string) *batch.Job { +func (s *Scanner) prepareKubeBenchJob(nodeItem core.Node) *batch.Job { + target := "node" + if _, ok := nodeItem.Labels[masterNodeLabel]; ok { + target = "master" + } return &batch.Job{ ObjectMeta: meta.ObjectMeta{ Name: uuid.New().String(), @@ -129,7 +129,7 @@ func (s *Scanner) prepareKubeBenchJob(nodeName string, target string) *batch.Job Spec: core.PodSpec{ RestartPolicy: core.RestartPolicyNever, HostPID: true, - NodeName: nodeName, + NodeName: nodeItem.Name, Volumes: []core.Volume{ { Name: "var-lib-etcd", diff --git a/pkg/kubehunter/crd/writer.go b/pkg/kubehunter/crd/writer.go index 43fe37c38..4223928e5 100644 --- a/pkg/kubehunter/crd/writer.go +++ b/pkg/kubehunter/crd/writer.go @@ -5,6 +5,8 @@ import ( "errors" "strings" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/aquasecurity/starboard/pkg/kube" "github.com/aquasecurity/starboard/pkg/kubehunter" @@ -25,14 +27,11 @@ func NewWriter(clientset starboardapi.Interface) kubehunter.Writer { } -func (w *writer) Write(ctx context.Context, report starboard.KubeHunterOutput, cluster string) (err error) { +func (w *writer) Write(ctx context.Context, report starboard.KubeHunterOutput, cluster string) error { if strings.TrimSpace(cluster) == "" { - err = errors.New("cluster name must not be blank") - return + return errors.New("cluster name must not be blank") } - // TODO Check if an instance of the report with the given name already exists. - // TODO If exists just update it, create new instance otherwise - _, err = w.clientset.AquasecurityV1alpha1().KubeHunterReports().Create(ctx, &starboard.KubeHunterReport{ + _, err := w.clientset.AquasecurityV1alpha1().KubeHunterReports().Create(ctx, &starboard.KubeHunterReport{ ObjectMeta: meta.ObjectMeta{ Name: cluster, Labels: map[string]string{ @@ -42,5 +41,15 @@ func (w *writer) Write(ctx context.Context, report starboard.KubeHunterOutput, c }, Report: report, }, meta.CreateOptions{}) - return + if err != nil && apierrors.IsAlreadyExists(err) { + found, err := w.clientset.AquasecurityV1alpha1().KubeHunterReports().Get(ctx, cluster, meta.GetOptions{}) + if err != nil { + return err + } + deepCopy := found.DeepCopy() + deepCopy.Report = report + _, err = w.clientset.AquasecurityV1alpha1().KubeHunterReports().Update(ctx, deepCopy, meta.UpdateOptions{}) + return err + } + return err }