Skip to content

Commit

Permalink
wrap-func
Browse files Browse the repository at this point in the history
  • Loading branch information
an1l4 committed Aug 21, 2023
1 parent f6f3785 commit edc2955
Show file tree
Hide file tree
Showing 7 changed files with 320 additions and 2 deletions.
20 changes: 18 additions & 2 deletions agent/kubviz/k8smetrics_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ func main() {
kubePreUpgradeChan := make(chan error, 1)
getAllResourceChan := make(chan error, 1)
trivyK8sMetricsChan := make(chan error, 1)
trivySbomcanChan := make(chan error, 1)
kubescoreMetricsChan := make(chan error, 1)
trivyImagescanChan := make(chan error, 1)
RakeesErrChan := make(chan error, 1)
Expand Down Expand Up @@ -133,6 +134,10 @@ func main() {
if err != nil {
log.Println(err)
}
case err := <-trivySbomcanChan:
if err != nil {
log.Println(err)
}
case err := <-trivyK8sMetricsChan:
if err != nil {
log.Println(err)
Expand All @@ -153,16 +158,19 @@ func main() {
go GetAllResources(config, js, &wg, getAllResourceChan)
go RakeesOutput(config, js, &wg, RakeesErrChan)
go getK8sEvents(clientset)
go RunTrivyImageScans(config, js, &wg, trivyImagescanChan)
//go RunTrivyImageScans(config, js, &wg, trivyImagescanChan)
go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan)
go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan)
//go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan)
go RunTrivyScans(config, js, &wg, trivySbomcanChan,trivyImagescanChan,trivyK8sMetricsChan)

wg.Wait()
// once the go routines completes we will close the error channels
close(outdatedErrChan)
close(kubePreUpgradeChan)
close(getAllResourceChan)
// close(clusterMetricsChan)
close(kubescoreMetricsChan)
close(trivySbomcanChan)
close(trivyImagescanChan)
close(trivyK8sMetricsChan)
close(RakeesErrChan)
Expand All @@ -183,6 +191,14 @@ func main() {
s.StartBlocking() // Blocks the main function
}

func RunTrivyScans(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup,trivySbomcanChan chan error, trivyImagescanChan chan error, trivyK8sMetricsChan chan error) {
defer wg.Done()
RunTrivySbomScan(config, js, wg, trivySbomcanChan)
RunTrivyImageScans(config, js, wg, trivyImagescanChan)
RunTrivyK8sClusterScan(wg, js, trivyK8sMetricsChan)

}

// publishMetrics publishes stream of events
// with subject "METRICS.created"
func publishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, errCh chan error) {
Expand Down
154 changes: 154 additions & 0 deletions agent/kubviz/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package main

import (
"context"
"encoding/json"
"fmt"
"log"
"os/exec"
"sync"

"github.com/google/uuid"
"github.com/intelops/kubviz/constants"
"github.com/intelops/kubviz/model"
"github.com/nats-io/nats.go"
"k8s.io/client-go/rest"
)

func publishTrivySbomReport(report model.Sbom, js nats.JetStreamContext, errCh chan error) {
metrics := model.Reports{
ID: uuid.New().String(),
Report: report,
}
metricsJson, _ := json.Marshal(metrics)
_, err := js.Publish(constants.TRIVY_SBOM_SUBJECT, metricsJson)
if err != nil {
errCh <- err
}

log.Printf("Trivy report with BomFormat:%v has been published\n", metrics.Report.BomFormat)
errCh <- nil
}

func executeCommandSbom(ctx context.Context, command string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "/bin/sh", "-c", command)
stdout, err := cmd.Output()

if err != nil {
log.Println("Execute Command Error", err.Error())
}

return stdout, nil
}

// func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
// defer wg.Done()
// images, err := ListImages(config)
// log.Println("length of images", len(images))

// if err != nil {
// log.Printf("failed to list images: %v", err)
// }

// ctx, cancel := context.WithTimeout(context.Background(), 700*time.Second)
// defer cancel()

// var wgc sync.WaitGroup
// wgc.Add(len(images)) // Set the wait group count to the number of images

// for i, image := range images {
// fmt.Printf("pullable Image %#v\n", image.PullableImage)

// // Start a goroutine for each image
// go func(i int, image model.RunningImage) {
// defer wgc.Done()

// // Execute the Trivy command with the context
// command := fmt.Sprintf("trivy image --format cyclonedx %s", image.PullableImage)
// out, err := executeCommandSbom(ctx, command)

// if ctx.Err() == context.DeadlineExceeded {
// log.Printf("Command execution timeout for image %s", image.PullableImage)
// return // Move on to the next image
// }

// if err != nil {
// log.Printf("Error executing Trivy for image %s: %v", image.PullableImage, err)
// return // Move on to the next image in case of an error
// }

// // Check if the output is empty or invalid JSON
// if len(out) == 0 {
// log.Printf("Trivy output is empty for image %s", image.PullableImage)
// return // Move on to the next image
// }

// // Extract the JSON data from the output
// var report model.Sbom
// err = json.Unmarshal(out, &report)
// if err != nil {
// log.Printf("Error unmarshaling JSON data for image %s: %v", image.PullableImage, err)
// return // Move on to the next image in case of an error
// }

// // Publish the report using the given function
// publishTrivySbomReport(report, js, errCh)
// }(i, image)
// }

// // Wait for all the goroutines to complete
// wgc.Wait()
// }
func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
defer wg.Done()
images, err := ListImages(config)
log.Println("length of images", len(images))

if err != nil {
log.Printf("failed to list images: %v", err)
}

ctx := context.Background()

var wgc sync.WaitGroup
wgc.Add(len(images)) // Set the wait group count to the number of images

for i, image := range images {
fmt.Printf("pullable Image %#v\n", image.PullableImage)

// Start a goroutine for each image
go func(i int, image model.RunningImage) {
defer wgc.Done()

// Execute the Trivy command with the context
//command := fmt.Sprintf("trivy image --format cyclonedx %s", image.PullableImage)
command := fmt.Sprintf("trivy -q image --format cyclonedx %s", image.PullableImage)
out, err := executeCommandSbom(ctx, command)

if err != nil {
log.Printf("Error executing Trivy for image %s: %v", image.PullableImage, err)
return // Move on to the next image in case of an error
}

// Check if the output is empty or invalid JSON
if len(out) == 0 {
log.Printf("Trivy output is empty for image %s", image.PullableImage)
return // Move on to the next image
}

// Extract the JSON data from the output
var report model.Sbom
err = json.Unmarshal(out, &report)
if err != nil {
log.Printf("Error unmarshaling JSON data for image %s: %v", image.PullableImage, err)
return // Move on to the next image in case of an error
}

// Publish the report using the given function
publishTrivySbomReport(report, js, errCh)
}(i, image)
}

// Wait for all the goroutines to complete
wgc.Wait()
}
46 changes: 46 additions & 0 deletions client/pkg/clickhouse/db_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type DBInterface interface {
InsertGitEvent(string)
InsertKubeScoreMetrics(model.KubeScoreRecommendations)
InsertTrivyImageMetrics(metrics model.TrivyImage)
InsertTrivySbomMetrics(metrics model.Reports)
InsertTrivyMetrics(metrics model.Trivy)
RetriveKetallEvent() ([]model.Resource, error)
RetriveOutdatedEvent() ([]model.CheckResultfinal, error)
Expand Down Expand Up @@ -431,6 +432,51 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) {

}
}
func (c *DBClient) InsertTrivySbomMetrics(metrics model.Reports) {
log.Println("####started inserting value")
result := metrics.Report
for _, com := range result.Components {
if len(result.Metadata.Tools) == 0 || len(com.Properties) == 0 || len(com.Hashes) == 0 || len(com.Licenses) == 0 {
continue
}
for _, depend := range result.Dependencies {
var (
tx, _ = c.conn.Begin()
stmt, _ = tx.Prepare(InsertTrivySbom)
)
if _, err := stmt.Exec(
metrics.ID,
result.Schema,
result.BomFormat,
result.SpecVersion,
result.SerialNumber,
result.Version,
result.Metadata.Timestamp,
result.Metadata.Tools[0].Vendor,
result.Metadata.Tools[0].Name,
result.Metadata.Tools[0].Version,
com.BomRef,
com.Type,
com.Name,
com.Version,
com.Properties[0].Name,
com.Properties[0].Value,
com.Hashes[0].Alg,
com.Hashes[0].Content,
com.Licenses[0].Expression,
com.Purl,
depend.Ref,
); err != nil {
log.Fatal(err)
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
stmt.Close()
}
}
log.Println("#### value inserted")
}
func (c *DBClient) Close() {
_ = c.conn.Close()
}
Expand Down
26 changes: 26 additions & 0 deletions client/pkg/clickhouse/statements.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,31 @@ const azureContainerPushEventTable DBStatement = `
SHAID String
) engine=File(TabSeparated)
`
const CreatetrivySbomTable DBStatement = `
CREATE TABLE IF NOT EXISTS trivysbom (
id UUID,
schema String,
bom_format String,
spec_version String,
serial_number String,
version INTEGER,
metadata_timestamp DateTime('UTC'),
metatool_vendor String,
metatool_name String,
metatool_version String,
component_bom_ref String,
component_type String,
component_name String,
component_version String,
component_property_name String,
component_property_value String,
component_hash_alg String,
component_hash_content String,
component_license_exp String,
component_purl String,
dependency_ref String
) engine=File(TabSeparated)
`

const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)"
const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)"
Expand All @@ -173,3 +198,4 @@ const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespac
const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"
const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
15 changes: 15 additions & 0 deletions client/pkg/clients/kubviz_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,21 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) {
log.Println()
},
},
{
Subject: constants.TRIVY_SBOM_SUBJECT,
Consumer: constants.Trivy_Sbom_Consumer,
Handler: func(msg *nats.Msg) {
msg.Ack()
var metrics model.Reports
err := json.Unmarshal(msg.Data, &metrics)
if err != nil {
log.Fatal(err)
}
log.Printf("Trivy sbom Metrics Received: %#v,", metrics)
conn.InsertTrivySbomMetrics(metrics)
log.Println()
},
},
{
Subject: constants.KubvizSubject,
Consumer: constants.KubvizConsumer,
Expand Down
2 changes: 2 additions & 0 deletions constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,6 @@ const (
TrivyConsumer = "TRIVY_CONSUMER"
TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage"
Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER"
TRIVY_SBOM_SUBJECT = "METRICS.trivysbom"
Trivy_Sbom_Consumer = "TRIVY_SBOM_CONSUMER"
)
59 changes: 59 additions & 0 deletions model/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package model

import (
"time"
)

type Reports struct {
ID string
Report Sbom
}

type Sbom struct {
Schema string `json:"$schema"`
BomFormat string `json:"bomFormat"`
SpecVersion string `json:"specVersion"`
SerialNumber string `json:"serialNumber"`
Version int `json:"version"`
Metadata struct {
Timestamp time.Time `json:"timestamp"`
Tools []struct {
Vendor string `json:"vendor"`
Name string `json:"name"`
Version string `json:"version"`
} `json:"tools"`
Component struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Purl string `json:"purl"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
} `json:"component"`
} `json:"metadata"`
Components []struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Version string `json:"version"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
Hashes []struct {
Alg string `json:"alg"`
Content string `json:"content"`
} `json:"hashes,omitempty"`
Licenses []struct {
Expression string `json:"expression"`
} `json:"licenses,omitempty"`
Purl string `json:"purl,omitempty"`
} `json:"components"`
Dependencies []struct {
Ref string `json:"ref"`
DependsOn []string `json:"dependsOn"`
} `json:"dependencies"`
Vulnerabilities []interface{} `json:"vulnerabilities"`
}

0 comments on commit edc2955

Please sign in to comment.