test(kube-bench): Add integration test for kube-bench command (aquase…

…curity#98)

Signed-off-by: Daniel Pacak <pacak.daniel@gmail.com>

.github/workflows/build.yaml

@@ -42,13 +42,15 @@ jobs:
           file: ./coverage.txt
       - name: Setup Kubernetes cluster (KIND)
         uses: engineerd/setup-kind@v0.4.0
+        with:
+          version: v0.8.1
       - name: Test connection to Kubernetes cluster
         run: |
           kubectl cluster-info
+          kubectl describe node
       - name: Run integration tests
         run: |
           make integration-tests
-          kubectl get crd
         env:
           KUBECONFIG: /home/runner/.kube/config
       - name: Release snapshot

Makefile

@@ -10,4 +10,4 @@ unit-tests: $(SOURCES)
 	go test -v -short -race -timeout 30s -coverprofile=coverage.txt -covermode=atomic ./...
 
 integration-tests: build
-	go test -v ./integration-tests
+	go test ./itest -ginkgo.v -ginkgo.progress -test.v

itest/integration_test.go

@@ -0,0 +1,134 @@
+package itest
+
+import (
+	"context"
+	"os/exec"
+	"time"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+
+	. "github.com/onsi/gomega/gbytes"
+
+	"github.com/aquasecurity/starboard/pkg/kube"
+	. "github.com/onsi/gomega/gstruct"
+
+	. "github.com/onsi/gomega/gexec"
+	apiextentions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	scanJobTimeout = 2 * time.Minute
+)
+
+var _ = Describe("Starboard CLI", func() {
+
+	BeforeEach(func() {
+		command := exec.Command(pathToStarboardCLI, []string{"init", "-v", "3"}...)
+		session, err := Start(command, GinkgoWriter, GinkgoWriter)
+		Expect(err).ToNot(HaveOccurred())
+		Eventually(session).Should(Exit(0))
+	})
+
+	Describe("Running init command", func() {
+		It("should initialize Starboard", func() {
+
+			crdList, err := apiextensionsClientset.CustomResourceDefinitions().List(context.TODO(), meta.ListOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			GetNames := func(crds []apiextentions.CustomResourceDefinition) []string {
+				names := make([]string, len(crds))
+				for i, crd := range crds {
+					names[i] = crd.Name
+				}
+				return names
+			}
+
+			Expect(crdList.Items).To(WithTransform(GetNames, ContainElements(
+				"ciskubebenchreports.aquasecurity.github.io",
+				"configauditreports.aquasecurity.github.io",
+				"kubehunterreports.aquasecurity.github.io",
+				"vulnerabilities.aquasecurity.github.io",
+			)))
+
+			_, err = kubernetesClientset.CoreV1().Namespaces().Get(context.TODO(), "starboard", meta.GetOptions{})
+			Expect(err).ToNot(HaveOccurred())
+
+			// TODO Assert other Kubernetes resources that we create in the init command
+		})
+	})
+
+	Describe("Running version command", func() {
+		It("should print the current version of the executable binary", func() {
+			command := exec.Command(pathToStarboardCLI, []string{"version"}...)
+			session, err := Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(session).Should(Say("Starboard Version: {Version:dev Commit:none Date:unknown}\n"))
+		})
+	})
+
+	Describe("Running kube-bench", func() {
+		It("should run kube-bench", func() {
+			command := exec.Command(pathToStarboardCLI, "kube-bench", "-v", "3")
+			session, err := Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(session, scanJobTimeout).Should(Exit(0))
+
+			nodeNames, err := GetNodeNames(context.TODO())
+			Expect(err).ToNot(HaveOccurred())
+
+			for _, nodeName := range nodeNames {
+				reportList, err := starboardClientset.AquasecurityV1alpha1().CISKubeBenchReports().List(context.TODO(), meta.ListOptions{
+					LabelSelector: labels.Set{
+						kube.LabelResourceKind: string(kube.KindNode),
+						kube.LabelResourceName: nodeName,
+					}.String(),
+				})
+				Expect(err).ToNot(HaveOccurred())
+				Expect(reportList.Items).To(HaveLen(1), "Expected CISKubeBenchReport for node %s but not found", nodeName)
+			}
+		})
+	})
+
+	// FIXME Figure out why kube-hunter is failing on GitHub actions runner, whereas it's fine with local KIND cluster
+	PDescribe("Running kube-hunter", func() {
+		It("should run kube-hunter", func() {
+			command := exec.Command(pathToStarboardCLI, "kube-hunter", "-v", "3")
+			session, err := Start(command, GinkgoWriter, GinkgoWriter)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(session, scanJobTimeout).Should(Exit(0))
+
+			report, err := starboardClientset.AquasecurityV1alpha1().KubeHunterReports().
+				Get(context.TODO(), "cluster", meta.GetOptions{})
+
+			Expect(err).ToNot(HaveOccurred())
+			Expect(report.Labels).To(MatchAllKeys(Keys{
+				kube.LabelResourceKind: Equal("Cluster"),
+				kube.LabelResourceName: Equal("cluster"),
+			}))
+		})
+	})
+
+	AfterEach(func() {
+		command := exec.Command(pathToStarboardCLI, []string{"cleanup", "-v", "3"}...)
+		session, err := Start(command, GinkgoWriter, GinkgoWriter)
+		Expect(err).ToNot(HaveOccurred())
+		Eventually(session).Should(Exit(0))
+
+		// TODO We have to wait for the termination of the starboard namespace. Otherwise the init command
+		// TODO run by the BeforeEach callback fails when it attempts to create Kubernetes objects in the
+		// TODO starboard namespace that is being terminated.
+		//
+		// TODO Maybe the cleanup command should block and wait unit the namespace is terminated?
+		Eventually(func() bool {
+			_, err := kubernetesClientset.CoreV1().Namespaces().Get(context.TODO(), "starboard", meta.GetOptions{})
+			return errors.IsNotFound(err)
+		}, 10*time.Second).Should(BeTrue())
+	})
+
+})

integration-tests/starboard_integration_suite_test.go → itest/suite_test.go

@@ -1,10 +1,15 @@
-package integration_tests
+package itest
 
 import (
+	"context"
 	"os"
 	"testing"
 
-	"github.com/onsi/gomega/gexec"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	starboardapi "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned"
+
+	. "github.com/onsi/gomega/gexec"
 	"k8s.io/client-go/tools/clientcmd"
 
 	. "github.com/onsi/ginkgo"
@@ -16,6 +21,7 @@ import (
 var (
 	kubernetesClientset    kubernetes.Interface
 	apiextensionsClientset apiextensions.ApiextensionsV1beta1Interface
+	starboardClientset     starboardapi.Interface
 )
 
 var (
@@ -24,7 +30,7 @@ var (
 
 var _ = BeforeSuite(func() {
 	var err error
-	pathToStarboardCLI, err = gexec.Build("github.com/aquasecurity/starboard/cmd/starboard")
+	pathToStarboardCLI, err = Build("github.com/aquasecurity/starboard/cmd/starboard")
 	Expect(err).ToNot(HaveOccurred())
 
 	config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG"))
@@ -35,6 +41,9 @@ var _ = BeforeSuite(func() {
 
 	apiextensionsClientset, err = apiextensions.NewForConfig(config)
 	Expect(err).ToNot(HaveOccurred())
+
+	starboardClientset, err = starboardapi.NewForConfig(config)
+	Expect(err).ToNot(HaveOccurred())
 })
 
 func TestStarboardCLI(t *testing.T) {
@@ -46,5 +55,17 @@ func TestStarboardCLI(t *testing.T) {
 }
 
 var _ = AfterSuite(func() {
-	gexec.CleanupBuildArtifacts()
+	CleanupBuildArtifacts()
 })
+
+func GetNodeNames(ctx context.Context) ([]string, error) {
+	nodesList, err := kubernetesClientset.CoreV1().Nodes().List(ctx, v1.ListOptions{})
+	if err != nil {
+		return nil, err
+	}
+	names := make([]string, len(nodesList.Items))
+	for i, node := range nodesList.Items {
+		names[i] = node.Name
+	}
+	return names, nil
+}

pkg/cmd/kube_bench.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"sync"
 
+	core "k8s.io/api/core/v1"
+
 	"github.com/aquasecurity/starboard/pkg/ext"
 	starboard "github.com/aquasecurity/starboard/pkg/generated/clientset/versioned"
 	"github.com/aquasecurity/starboard/pkg/kubebench"
@@ -16,10 +18,6 @@ import (
 	"k8s.io/klog"
 )
 
-const (
-	masterNodeLabel = "node-role.kubernetes.io/master"
-)
-
 func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "kube-bench",
@@ -42,33 +40,35 @@ func NewKubeBenchCmd(cf *genericclioptions.ConfigFlags) *cobra.Command {
 			if err != nil {
 				return
 			}
-			// List Nodes
 			nodeList, err := kubernetesClientset.CoreV1().Nodes().List(ctx, meta.ListOptions{})
 			if err != nil {
-				err = fmt.Errorf("list nodes: %w", err)
+				err = fmt.Errorf("listing nodes: %w", err)
 				return
 			}
+			scanner := kubebench.NewScanner(opts, kubernetesClientset)
+			writer := crd.NewWriter(ext.NewSystemClock(), starboardClientset)
+
 			var wg sync.WaitGroup
-			wg.Add(len(nodeList.Items))
-			for _, nodeItem := range nodeList.Items {
-				target := "node"
-				if _, ok := nodeItem.Labels[masterNodeLabel]; ok {
-					target = "master"
-				}
-				nodeName := nodeItem.Name
-				go func() {
-					klog.V(3).Infof("Node name: %s Label:%s", nodeName, target)
-					report, node, err := kubebench.NewScanner(opts, kubernetesClientset).Scan(ctx, nodeName, target, &wg)
+
+			for _, node := range nodeList.Items {
+				wg.Add(1)
+				go func(node core.Node) {
+					defer wg.Done()
+
+					report, err := scanner.Scan(ctx, node)
 
 					if err != nil {
-						klog.Warningf("Node name: %s Error NewScanner: %s", nodeName, err)
+						klog.Errorf("Error while running kube-bench on node: %s: %v", node.Name, err)
+						return
 					}
-					err = crd.NewWriter(ext.NewSystemClock(), starboardClientset).Write(ctx, report, node)
+					err = writer.Write(ctx, report, &node)
 					if err != nil {
-						klog.Warningf("Node name: %s Error NewWriter: %s", nodeName, err)
+						klog.Errorf("Error while writing kube-bench report for node: %s: %v", node.Name, err)
+						return
 					}
-				}()
+				}(node)
 			}
+
 			wg.Wait()
 			return
 		},