kubescore

agent/kubviz/kube_score.go

@@ -3,15 +3,17 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"log"
+	exec "os/exec"
+	"sync"
+
 	"github.com/google/uuid"
 	"github.com/intelops/kubviz/constants"
 	"github.com/intelops/kubviz/model"
 	"github.com/nats-io/nats.go"
+	"github.com/zegl/kube-score/renderer/json_v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"log"
-	exec "os/exec"
-	"sync"
 )
 
 func RunKubeScore(clientset *kubernetes.Clientset, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
@@ -26,41 +28,52 @@ func RunKubeScore(clientset *kubernetes.Clientset, js nats.JetStreamContext, wg
 	}
 
 	log.Printf("Namespace size: %d", len(nsList.Items))
+	var wgNamespaces sync.WaitGroup
 	for _, n := range nsList.Items {
+		wgNamespaces.Add(1)
 		log.Printf("Publishing kube-score recommendations for namespace: %s\n", n.Name)
-		publish(n.Name, js, errCh)
+		go publish(n.Name, js, &wgNamespaces, errCh)
 	}
 }
 
-func publish(ns string, js nats.JetStreamContext, errCh chan error) {
-	cmd := "kubectl api-resources --verbs=list --namespaced -o name | xargs -n1 -I{} sh -c \"kubectl get {} -n " + ns + " -oyaml && echo ---\" | kube-score score - "
+func publish(ns string, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
+	defer wg.Done()
+	var report []json_v2.ScoredObject
+
+	cmd := "kubectl api-resources --verbs=list --namespaced -o name | xargs -n1 -I{} sh -c \"kubectl get {} -n " + ns + " -oyaml && echo ---\" | kube-score score - -o json"
 	log.Printf("Command:  %#v,", cmd)
 	out, err := executeCommand(cmd)
+
+	err = json.Unmarshal([]byte(out), &report)
+	if err != nil {
+		log.Printf("Error occurred while Unmarshalling json: %v", err)
+		errCh <- err
+	}
+
 	if err != nil {
 		log.Println("Error occurred while running kube-score: ", err)
 		errCh <- err
 	}
-	err = publishKubescoreMetrics(uuid.New().String(), ns, out, js)
+	err = publishKubescoreMetrics(uuid.New().String(), report, js)
 	if err != nil {
 		errCh <- err
 	}
 	errCh <- nil
 }
 
-func publishKubescoreMetrics(id string, ns string, recommendations string, js nats.JetStreamContext) error {
+func publishKubescoreMetrics(id string, report []json_v2.ScoredObject, js nats.JetStreamContext) error {
 	metrics := model.KubeScoreRecommendations{
-		ID:              id,
-		Namespace:       ns,
-		Recommendations: recommendations,
-		ClusterName:     ClusterName,
+		ID:          id,
+		ClusterName: ClusterName,
+		Report:      report,
 	}
 	metricsJson, _ := json.Marshal(metrics)
 	_, err := js.Publish(constants.KUBESCORE_SUBJECT, metricsJson)
 	if err != nil {
 		return err
 	}
 	log.Printf("Recommendations with ID:%s has been published\n", id)
-	log.Printf("Recommendations  :%#v", recommendations)
+	log.Printf("Recommendations  :%#v", report)
 	return nil
 }
 

client/pkg/clickhouse/db_client.go

@@ -269,24 +269,48 @@ func (c *DBClient) InsertContainerEvent(event string) {
 }
 
 func (c *DBClient) InsertKubeScoreMetrics(metrics model.KubeScoreRecommendations) {
-	var (
-		tx, _   = c.conn.Begin()
-		stmt, _ = tx.Prepare(InsertKubeScore)
-	)
-	defer stmt.Close()
-	if _, err := stmt.Exec(
-		metrics.ID,
-		metrics.Namespace,
-		metrics.ClusterName,
-		metrics.Recommendations,
-	); err != nil {
+	tx, err := c.conn.Begin()
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer tx.Rollback()
+
+	stmt, err := tx.Prepare(InsertKubeScore)
+	if err != nil {
 		log.Fatal(err)
 	}
+	defer stmt.Close()
+
+	for _, result := range metrics.Report {
+		for _, check := range result.Checks {
+			for _, comments := range check.Comments {
+
+				if _, err := stmt.Exec(
+					metrics.ID,
+					metrics.ClusterName,
+					result.ObjectName,
+					result.TypeMeta.Kind,
+					result.TypeMeta.APIVersion,
+					result.ObjectMeta.Name,
+					result.ObjectMeta.Namespace,
+					check.Check.TargetType,
+					comments.Description,
+					comments.Path,
+					comments.Summary,
+					result.FileName,
+					result.FileRow,
+				); err != nil {
+					log.Println("Error while inserting KubeScore metrics:", err)
+				}
+			}
+
+		}
+	}
+	// Commit the transaction after the loop finishes.
 	if err := tx.Commit(); err != nil {
 		log.Fatal(err)
 	}
 }
-
 func (c *DBClient) InsertTrivyMetrics(metrics model.Trivy) {
 	for _, finding := range metrics.Report.Findings {
 		for _, result := range finding.Results {

client/pkg/clickhouse/statements.go

@@ -71,14 +71,24 @@ CREATE TABLE IF NOT EXISTS outdated_images (
 	VersionsBehind Int64
 ) engine=File(TabSeparated)
 `
-const kubescoreTable DBStatement = `
-	    CREATE TABLE IF NOT EXISTS kubescore (
-		    id UUID,
-			namespace String,
-			cluster_name String,
-			recommendations String
-	    ) engine=File(TabSeparated)
-	`
+const kubescoreTable DBStatement = ` 
+CREATE TABLE IF NOT EXISTS kubescore (
+	id UUID,
+	clustername String,
+	object_name String,
+	kind String,
+	apiVersion String,
+	name String,
+	namespace String,
+	target_type String,
+	description String,
+	path String,
+	summary String,
+	file_name String,
+	file_row  BIGINT
+) engine=File(TabSeparated)
+`
+
 const trivyTableVul DBStatement = `
 	    CREATE TABLE IF NOT EXISTS trivy_vul (
 		    id UUID,
@@ -156,7 +166,7 @@ const InsertKubvizEvent DBStatement = "INSERT INTO events (ClusterName, Id, Even
 const clickhouseExperimental DBStatement = `SET allow_experimental_object_type=1;`
 const containerDockerhubTable DBStatement = `CREATE table IF NOT EXISTS container_dockerhub(event JSON) ENGINE = MergeTree ORDER BY tuple();`
 const containerGithubTable DBStatement = `CREATE table IF NOT EXISTS container_github(event JSON) ENGINE = MergeTree ORDER BY tuple();`
-const InsertKubeScore string = "INSERT INTO kubescore (id, namespace, cluster_name, recommendations) VALUES (?, ?, ?, ?)"
+const InsertKubeScore string = "INSERT INTO kubescore(id,clustername,object_name,kind,apiVersion,name,namespace,target_type,description,path,summary,file_name,file_row) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)"
 const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespace, kind, name, vul_id, vul_vendor_ids, vul_pkg_id, vul_pkg_name, vul_pkg_path, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?)"
 const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id,  vul_pkg_id, vul_pkg_name,  vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
 const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/vijeyash1/go-github-container v1.0.0
+	github.com/zegl/kube-score v1.17.0
 	golang.org/x/term v0.10.0
 	k8s.io/api v0.27.3
 	k8s.io/apimachinery v0.27.3
@@ -49,7 +50,7 @@ require (
 	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.1 // indirect
 	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/fatih/color v1.14.1 // indirect
+	github.com/fatih/color v1.15.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect

go.sum

@@ -118,8 +118,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
 github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM=
-github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
-github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
+github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
+github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
 github.com/fernet/fernet-go v0.0.0-20180830025343-9eac43b88a5e/go.mod h1:2H9hjfbpSMHwY503FclkV/lZTBh2YlOmLLSda12uL8c=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
@@ -459,6 +459,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/zclconf/go-cty v1.10.0 h1:mp9ZXQeIcN8kAwuqorjH+Q+njbJKjLrvB2yIh4q7U+0=
 github.com/zclconf/go-cty-yaml v1.0.2 h1:dNyg4QLTrv2IfJpm7Wtxi55ed5gLGOlPrZ6kMd51hY0=
+github.com/zegl/kube-score v1.17.0 h1:vedzK0pm5yOb1ocm5gybMNYsJRG8iTAatbo3LFIWbUc=
+github.com/zegl/kube-score v1.17.0/go.mod h1:0pt4Lt36uTKPiCQbXQFow29eaAbgMLI9RoESjBoGSq0=
 go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
 go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=

model/kubescore.go

@@ -1,8 +1,9 @@
 package model
 
+import "github.com/zegl/kube-score/renderer/json_v2"
+
 type KubeScoreRecommendations struct {
-	ID              string
-	Namespace       string
-	Recommendations string
-	ClusterName     string
+	ID          string
+	ClusterName string
+	Report      []json_v2.ScoredObject
 }