Feat/add compliance check default status (aquasecurity#1030)

deploy/crd/clustercompliancereports.crd.yaml

@@ -114,6 +114,13 @@ spec:
                           - MEDIUM
                           - LOW
                           - UNKNOWN
+                      defaultStatus:
+                        type: string
+                        description: 'define the default value for check status in case resource not found'
+                        enum:
+                          - PASS
+                          - WARN
+                          - FAIL
             status:
               x-kubernetes-preserve-unknown-fields: true
               type: object

deploy/specs/nsa-1.0.yaml

@@ -139,6 +139,7 @@ spec:
       id: '1.12'
       kinds:
         - NetworkPolicy
+      defaultStatus: 'FAIL'
       mapping:
         scanner: config-audit
         checks:
@@ -169,6 +170,7 @@ spec:
       id: '4.0'
       kinds:
         - ResourceQuota
+      defaultStatus: 'FAIL'
       mapping:
         scanner: config-audit
         checks:
@@ -281,9 +283,10 @@ spec:
       description: 'Control check whether service mesh is used in cluster'
       id: '9.0'
       kinds:
-        - Node
+        - IstioOperator
+      defaultStatus: 'FAIL'
       mapping:
-        scanner: kube-bench
+        scanner: config-audit
         checks:
           - id: "<check need to be added>"
       severity: 'MEDIUM'

deploy/static/starboard.yaml

@@ -521,6 +521,13 @@ spec:
                           - MEDIUM
                           - LOW
                           - UNKNOWN
+                      defaultStatus:
+                        type: string
+                        description: 'define the default value for check status in case resource not found'
+                        enum:
+                          - PASS
+                          - WARN
+                          - FAIL
             status:
               x-kubernetes-preserve-unknown-fields: true
               type: object
@@ -2110,6 +2117,7 @@ spec:
       id: '1.12'
       kinds:
         - NetworkPolicy
+      defaultStatus: 'FAIL'
       mapping:
         scanner: config-audit
         checks:
@@ -2140,6 +2148,7 @@ spec:
       id: '4.0'
       kinds:
         - ResourceQuota
+      defaultStatus: 'FAIL'
       mapping:
         scanner: config-audit
         checks:
@@ -2252,9 +2261,11 @@ spec:
       description: 'Control check whether service mesh is used in cluster'
       id: '9.0'
       kinds:
-        - Node
+        - IstioOperator
+      defaultStatus: 'FAIL'
       mapping:
-        scanner: kube-bench
+        scanner: config-audit
         checks:
           - id: "<check need to be added>"
       severity: 'MEDIUM'
+

pkg/apis/aquasecurity/v1alpha1/compliance_types.go

@@ -31,12 +31,13 @@ type ReportSpec struct {
 
 //Control represent the cps controls data and mapping checks
 type Control struct {
-	ID          string   `json:"id"`
-	Name        string   `json:"name"`
-	Description string   `json:"description,,omitempty"`
-	Kinds       []string `json:"kinds"`
-	Mapping     Mapping  `json:"mapping"`
-	Severity    Severity `json:"severity"`
+	ID            string        `json:"id"`
+	Name          string        `json:"name"`
+	Description   string        `json:"description,omitempty"`
+	Kinds         []string      `json:"kinds"`
+	Mapping       Mapping       `json:"mapping"`
+	Severity      Severity      `json:"severity"`
+	DefaultStatus ControlStatus `json:"defaultStatus,omitempty"`
 }
 
 //SpecCheck represent the scanner who perform the control check
@@ -74,3 +75,11 @@ type ControlCheck struct {
 	FailTotal   int      `json:"failTotal"`
 	Severity    Severity `json:"severity"`
 }
+
+type ControlStatus string
+
+const (
+	FailStatus ControlStatus = "FAIL"
+	PassStatus ControlStatus = "PASS"
+	WarnStatus ControlStatus = "WARN"
+)

pkg/apis/aquasecurity/v1alpha1/compliancedetail_types.go

@@ -40,16 +40,16 @@ type ControlCheckDetails struct {
 }
 
 type ResultDetails struct {
-	Name      string `json:"name"`
-	Namespace string `json:"namespace"`
-	Msg       string `json:"msg"`
-	Status    string `json:"status"`
+	Name      string        `json:"name,omitempty"`
+	Namespace string        `json:"namespace,omitempty"`
+	Msg       string        `json:"msg"`
+	Status    ControlStatus `json:"status"`
 }
 
 type ScannerCheckResult struct {
 	ObjectType  string          `json:"objectType"`
-	ID          string          `json:"id"`
-	Remediation string          `json:"remediation"`
+	ID          string          `json:"id,omitempty"`
+	Remediation string          `json:"remediation,omitempty"`
 	Details     []ResultDetails `json:"details"`
 }
 

pkg/compliance/clustercompliancereport_test.go

@@ -100,7 +100,7 @@ var _ = ginkgo.Describe("cluster compliance report", func() {
 		})
 
 		ginkgo.It("check compliance compliance report status is updated following to changes occur with cis-bench and config-audit report", func() {
-			// update cis-benchmark report with failed tests and compare update compliance report
+			// update cis-benchmark report and config-audit with failed tests and compare update compliance report
 			var updatedCisBench v1alpha1.CISKubeBenchReport
 			err = loadResource("./testdata/fixture/cisBenchmarkReportUpdate.json", &updatedCisBench)
 			Expect(err).ToNot(HaveOccurred())
@@ -126,10 +126,23 @@ var _ = ginkgo.Describe("cluster compliance report", func() {
 			Expect(err).ToNot(HaveOccurred())
 			sort.Sort(controlSort(complianceReportUpdate.Status.ControlChecks))
 			sort.Sort(controlSort(clusterComplianceReportUpdate.Status.ControlChecks))
-
 			// validate updated cluster compliance report status
 			Expect(cmp.Equal(complianceReportUpdate.Status, clusterComplianceReportUpdate.Status, ignoreTimeStamp())).To(BeTrue())
 		})
+		ginkgo.It("check compliance compliance report detail is updated following to changes occur with cis-bench and config-audit report", func() {
+			// update cis-benchmark report and config-audit with failed tests and compare update compliance report
+			var clusterComplianceDetialReport v1alpha1.ClusterComplianceDetailReport
+			err = loadResource("./testdata/fixture/clusterComplianceDetailReportUpdate.json", &clusterComplianceDetialReport)
+			complianceDetailReport, err := getDetailReport(context.TODO(), types.NamespacedName{Namespace: "", Name: "nsa-details"}, client)
+			Expect(err).ToNot(HaveOccurred())
+			sort.Sort(controlDetailSort(complianceDetailReport.Report.ControlChecks))
+			sort.Sort(controlDetailSort(clusterComplianceDetialReport.Report.ControlChecks))
+			for i := 0; i < len(complianceDetailReport.Report.ControlChecks); i++ {
+				sort.Sort(controlObjectTypeSort(complianceDetailReport.Report.ControlChecks[i].ScannerCheckResult))
+				sort.Sort(controlObjectTypeSort(clusterComplianceDetialReport.Report.ControlChecks[i].ScannerCheckResult))
+			}
+			Expect(cmp.Equal(complianceDetailReport.Report, clusterComplianceDetialReport.Report, ignoreTimeStamp())).To(BeTrue())
+		})
 	})
 
 	ginkgo.Context("reconcile compliance spec report without cis-bench and audit-config data and validate compliance reports data", func() {

pkg/compliance/io.go

@@ -15,6 +15,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	ResourceDoNotExistInCluster = "Resource do not exist in cluster"
+)
+
 type Mgr interface {
 	GenerateComplianceReport(ctx context.Context, spec v1alpha1.ReportSpec) (*v1alpha1.ClusterComplianceReport, error)
 }
@@ -40,6 +44,7 @@ type specDataMapping struct {
 	scannerResourceListNames map[string]*hashset.Set
 	controlIDControlObject   map[string]v1alpha1.Control
 	controlCheckIds          map[string][]string
+	controlIdResources       map[string][]string
 }
 
 func (w *cm) GenerateComplianceReport(ctx context.Context, spec v1alpha1.ReportSpec) (*v1alpha1.ClusterComplianceReport, error) {
@@ -145,6 +150,9 @@ func (w *cm) getTotals(controlChecks []v1alpha1.ControlCheck) summaryTotal {
 // controlChecksByScannerChecks build control checks list by parsing test results and mapping it to relevant scanner
 func (w *cm) controlChecksByScannerChecks(smd *specDataMapping, checkIdsToResults map[string][]*ScannerCheckResult) []v1alpha1.ControlCheck {
 	controlChecks := make([]v1alpha1.ControlCheck, 0)
+	if len(checkIdsToResults) == 0 {
+		return controlChecks
+	}
 	for controlID, checkIds := range smd.controlCheckIds {
 		var passTotal, failTotal, total int
 		for _, checkId := range checkIds {
@@ -153,9 +161,9 @@ func (w *cm) controlChecksByScannerChecks(smd *specDataMapping, checkIdsToResult
 				for _, checkResult := range results {
 					for _, crd := range checkResult.Details {
 						switch crd.Status {
-						case Pass, Warn:
+						case v1alpha1.PassStatus, v1alpha1.WarnStatus:
 							passTotal++
-						case Fail:
+						case v1alpha1.FailStatus:
 							failTotal++
 						}
 						total++
@@ -165,6 +173,14 @@ func (w *cm) controlChecksByScannerChecks(smd *specDataMapping, checkIdsToResult
 		}
 		control, ok := smd.controlIDControlObject[controlID]
 		if ok {
+			if passTotal == 0 && failTotal == 0 {
+				if control.DefaultStatus == v1alpha1.FailStatus {
+					failTotal = 1
+				}
+				if control.DefaultStatus == v1alpha1.PassStatus {
+					passTotal = 1
+				}
+			}
 			controlChecks = append(controlChecks, v1alpha1.ControlCheck{ID: controlID,
 				Name:        control.Name,
 				Description: control.Description,
@@ -179,22 +195,22 @@ func (w *cm) controlChecksByScannerChecks(smd *specDataMapping, checkIdsToResult
 // controlChecksDetailsByScannerChecks build control checks with details list by parsing test results and mapping it to relevant tool
 func (w *cm) controlChecksDetailsByScannerChecks(smd *specDataMapping, checkIdsToResults map[string][]*ScannerCheckResult) []v1alpha1.ControlCheckDetails {
 	controlChecks := make([]v1alpha1.ControlCheckDetails, 0)
+	if len(checkIdsToResults) == 0 {
+		return controlChecks
+	}
 	for controlID, checkIds := range smd.controlCheckIds {
 		control, ok := smd.controlIDControlObject[controlID]
 		if ok {
 			for _, checkId := range checkIds {
 				results, ok := checkIdsToResults[checkId]
+				ctta := make([]v1alpha1.ScannerCheckResult, 0)
 				if ok {
-					ctta := make([]v1alpha1.ScannerCheckResult, 0)
-					for _, checkResult := range results {
-						var ctt v1alpha1.ScannerCheckResult
-						rds := make([]v1alpha1.ResultDetails, 0)
-						for _, crd := range checkResult.Details {
-							rds = append(rds, v1alpha1.ResultDetails{Name: crd.Name, Namespace: crd.Namespace, Msg: crd.Msg, Status: crd.Status})
-						}
-						ctt = v1alpha1.ScannerCheckResult{ID: checkResult.ID, ObjectType: checkResult.ObjectType, Remediation: checkResult.Remediation, Details: rds}
-						ctta = append(ctta, ctt)
-					}
+					scr := w.createScanCheckResult(results)
+					ctta = append(ctta, scr...)
+				} else {
+					w.createDefaultScanResult(smd, control, controlID, &ctta)
+				}
+				if len(ctta) > 0 {
 					controlChecks = append(controlChecks, v1alpha1.ControlCheckDetails{ID: controlID,
 						Name:               control.Name,
 						Description:        control.Description,
@@ -207,6 +223,36 @@ func (w *cm) controlChecksDetailsByScannerChecks(smd *specDataMapping, checkIdsT
 	return controlChecks
 }
 
+func (w *cm) createDefaultScanResult(smd *specDataMapping, control v1alpha1.Control, controlID string, ctta *[]v1alpha1.ScannerCheckResult) {
+	if control.DefaultStatus == v1alpha1.FailStatus {
+		resources := smd.controlIdResources[controlID]
+		for _, resource := range resources {
+			ctt := v1alpha1.ScannerCheckResult{ObjectType: resource, Details: []v1alpha1.ResultDetails{{Msg: ResourceDoNotExistInCluster, Status: v1alpha1.FailStatus}}}
+			*ctta = append(*ctta, ctt)
+		}
+	}
+}
+
+func (w *cm) createScanCheckResult(results []*ScannerCheckResult) []v1alpha1.ScannerCheckResult {
+	ctta := make([]v1alpha1.ScannerCheckResult, 0)
+	for _, checkResult := range results {
+		var ctt v1alpha1.ScannerCheckResult
+		rds := make([]v1alpha1.ResultDetails, 0)
+		for _, crd := range checkResult.Details {
+			//control check detail relevant to fail checks only
+			if crd.Status == v1alpha1.PassStatus || crd.Status == v1alpha1.WarnStatus {
+				continue
+			}
+			rds = append(rds, v1alpha1.ResultDetails{Name: crd.Name, Namespace: crd.Namespace, Msg: crd.Msg, Status: crd.Status})
+		}
+		if len(rds) > 0 {
+			ctt = v1alpha1.ScannerCheckResult{ID: checkResult.ID, ObjectType: checkResult.ObjectType, Remediation: checkResult.Remediation, Details: rds}
+			ctta = append(ctta, ctt)
+		}
+	}
+	return ctta
+}
+
 func (w *cm) checkIdsToResults(scannerResourceMap map[string]map[string]client.ObjectList) (map[string][]*ScannerCheckResult, error) {
 	checkIdsToResults := make(map[string][]*ScannerCheckResult)
 	for scanner, resourceListMap := range scannerResourceMap {
@@ -238,13 +284,19 @@ func (w *cm) populateSpecDataToMaps(spec v1alpha1.ReportSpec) *specDataMapping {
 	controlCheckIds := make(map[string][]string)
 	//scanner to resource list map
 	scannerResourceListName := make(map[string]*hashset.Set)
+	//controlOID to resources
+	controlIdResources := make(map[string][]string)
 	for _, control := range spec.Controls {
 		control.Kinds = mapKinds(control)
 		if _, ok := scannerResourceListName[control.Mapping.Scanner]; !ok {
 			scannerResourceListName[control.Mapping.Scanner] = hashset.New()
 		}
+		if _, ok := controlIdResources[control.ID]; !ok {
+			controlIdResources[control.ID] = make([]string, 0)
+		}
 		for _, resource := range control.Kinds {
 			scannerResourceListName[control.Mapping.Scanner].Add(resource)
+			controlIdResources[control.ID] = append(controlIdResources[control.ID], resource)
 		}
 		controlIDControlObject[control.ID] = control
 		//update control resource list map
@@ -254,9 +306,11 @@ func (w *cm) populateSpecDataToMaps(spec v1alpha1.ReportSpec) *specDataMapping {
 			}
 			controlCheckIds[control.ID] = append(controlCheckIds[control.ID], check.ID)
 		}
+
 	}
 	return &specDataMapping{
 		scannerResourceListNames: scannerResourceListName,
 		controlIDControlObject:   controlIDControlObject,
-		controlCheckIds:          controlCheckIds}
+		controlCheckIds:          controlCheckIds,
+		controlIdResources:       controlIdResources}
 }

pkg/compliance/io_test.go

@@ -77,10 +77,11 @@ func TestControlChecksByScannerChecks(t *testing.T) {
 		mapScannerResult map[string][]*ScannerCheckResult
 		want             []v1alpha1.ControlCheck
 	}{
-		{name: " control checks by scanner checks", specPath: "./testdata/fixture/nsa-1.0.yaml", want: []v1alpha1.ControlCheck{{ID: "1.0", Name: "Non-root containers", PassTotal: 1, FailTotal: 0, Severity: "MEDIUM"}, {ID: "8.1", Name: "Audit log path is configure", PassTotal: 0, FailTotal: 1, Severity: "MEDIUM"}},
+		{name: " control checks by scanner checks", specPath: "./testdata/fixture/nsa-1.0.yaml", want: []v1alpha1.ControlCheck{{ID: "1.0", Name: "Non-root containers",
+			PassTotal: 1, FailTotal: 0, Severity: "MEDIUM"}, {ID: "8.1", Name: "Audit log path is configure", PassTotal: 0, FailTotal: 1, Severity: "MEDIUM"}},
 			mapScannerResult: map[string][]*ScannerCheckResult{
-				"KSV012": {{ID: "1.0", Remediation: "aaa", Details: []ResultDetails{{Status: "pass"}}}},
-				"1.2.22": {{ID: "2.0", Remediation: "bbb", Details: []ResultDetails{{Status: "fail"}}}},
+				"KSV012": {{ID: "1.0", Remediation: "aaa", Details: []ResultDetails{{Status: "PASS"}}}},
+				"1.2.22": {{ID: "2.0", Remediation: "bbb", Details: []ResultDetails{{Status: "FAIL"}}}},
 			}}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -130,7 +131,7 @@ func TestCheckIdsToResults(t *testing.T) {
 		reportList map[string]map[string]client.ObjectList
 		wantResult map[string][]*ScannerCheckResult
 	}{
-		{name: "map check ids to results report", reportList: map[string]map[string]client.ObjectList{ConfigAudit: {"Pod": getConfAudit([]string{"KSV037", "KSV038"}, []bool{true, false}, []string{"aaa", "bbb"})}, KubeBench: {"Node": getCisInstance([]string{"1.1", "2.2"}, []string{"Pass", "Fail"}, []string{"aaa", "bbb"})}}, wantResult: getWantMapResults("./testdata/fixture/check_data_result.json")},
+		{name: "map check ids to results report", reportList: map[string]map[string]client.ObjectList{ConfigAudit: {"Pod": getConfAudit([]string{"KSV037", "KSV038"}, []bool{true, false}, []string{"aaa", "bbb"})}, KubeBench: {"Node": getCisInstance([]string{"1.1", "2.2"}, []string{"PASS", "FAIL"}, []string{"aaa", "bbb"})}}, wantResult: getWantMapResults("./testdata/fixture/check_data_result.json")},
 		{name: "map empty data ", reportList: map[string]map[string]client.ObjectList{}, wantResult: map[string][]*ScannerCheckResult{}},
 	}
 	for _, tt := range tests {