Skip to content

Commit

Permalink
testing1
Browse files Browse the repository at this point in the history
  • Loading branch information
an1l4 committed Aug 21, 2023
1 parent f6f3785 commit f1b7648
Show file tree
Hide file tree
Showing 7 changed files with 266 additions and 2 deletions.
20 changes: 18 additions & 2 deletions agent/kubviz/k8smetrics_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func main() {
trivyK8sMetricsChan := make(chan error, 1)
kubescoreMetricsChan := make(chan error, 1)
trivyImagescanChan := make(chan error, 1)
trivySbomcanChan := make(chan error, 1)
RakeesErrChan := make(chan error, 1)
// Start a goroutine to handle errors
doneChan := make(chan bool)
Expand Down Expand Up @@ -133,6 +134,10 @@ func main() {
if err != nil {
log.Println(err)
}
case err := <-trivySbomcanChan:
if err != nil {
log.Println(err)
}
case err := <-trivyK8sMetricsChan:
if err != nil {
log.Println(err)
Expand All @@ -153,9 +158,11 @@ func main() {
go GetAllResources(config, js, &wg, getAllResourceChan)
go RakeesOutput(config, js, &wg, RakeesErrChan)
go getK8sEvents(clientset)
go RunTrivyImageScans(config, js, &wg, trivyImagescanChan)
//go RunTrivyImageScans(config, js, &wg, trivyImagescanChan)
go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan)
go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan)
//go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan)
go RunTrivyScans(config, js, &wg, trivySbomcanChan,trivyImagescanChan,trivyK8sMetricsChan)

wg.Wait()
// once the go routines completes we will close the error channels
close(outdatedErrChan)
Expand All @@ -164,6 +171,7 @@ func main() {
// close(clusterMetricsChan)
close(kubescoreMetricsChan)
close(trivyImagescanChan)
close(trivySbomcanChan)
close(trivyK8sMetricsChan)
close(RakeesErrChan)
// Signal that all other goroutines have finished
Expand All @@ -183,6 +191,14 @@ func main() {
s.StartBlocking() // Blocks the main function
}

func RunTrivyScans(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup,trivySbomcanChan chan error, trivyImagescanChan chan error, trivyK8sMetricsChan chan error) {
defer wg.Done()
RunTrivySbomScan(config, js, wg, trivySbomcanChan)
RunTrivyImageScans(config, js, wg, trivyImagescanChan)
RunTrivyK8sClusterScan(wg, js, trivyK8sMetricsChan)

}

// publishMetrics publishes stream of events
// with subject "METRICS.created"
func publishMetrics(clientset *kubernetes.Clientset, js nats.JetStreamContext, errCh chan error) {
Expand Down
102 changes: 102 additions & 0 deletions agent/kubviz/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package main

import (
"context"
"encoding/json"
"fmt"
"log"
"os/exec"
"sync"
"time"

"github.com/google/uuid"
"github.com/intelops/kubviz/constants"
"github.com/intelops/kubviz/model"
"github.com/nats-io/nats.go"
"k8s.io/client-go/rest"
)

func publishTrivySbomReport(report model.Sbom, js nats.JetStreamContext, errCh chan error) {
metrics := model.Reports{
ID: uuid.New().String(),
Report: report,
}
metricsJson, _ := json.Marshal(metrics)
_, err := js.Publish(constants.TRIVY_SBOM_SUBJECT, metricsJson)
if err != nil {
errCh <- err
}

log.Printf("Trivy report with BomFormat:%v has been published\n", metrics.Report.BomFormat)
errCh <- nil
}

func executeCommandSbom(ctx context.Context, command string) ([]byte, error) {
cmd := exec.CommandContext(ctx, "/bin/sh", "-c", command)
stdout, err := cmd.Output()

if err != nil {
log.Println("Execute Command Error", err.Error())
}

return stdout, nil
}

func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
defer wg.Done()
images, err := ListImages(config)
log.Println("length of images", len(images))

if err != nil {
log.Printf("failed to list images: %v", err)
}

ctx, cancel := context.WithTimeout(context.Background(), 700*time.Second)
defer cancel()

var wgc sync.WaitGroup
wgc.Add(len(images)) // Set the wait group count to the number of images

for i, image := range images {
fmt.Printf("pullable Image %#v\n", image.PullableImage)

// Start a goroutine for each image
go func(i int, image model.RunningImage) {
defer wgc.Done()

// Execute the Trivy command with the context
command := fmt.Sprintf("trivy image --format cyclonedx %s", image.PullableImage)
out, err := executeCommandSbom(ctx, command)

if ctx.Err() == context.DeadlineExceeded {
log.Printf("Command execution timeout for image %s", image.PullableImage)
return // Move on to the next image
}

if err != nil {
log.Printf("Error executing Trivy for image %s: %v", image.PullableImage, err)
return // Move on to the next image in case of an error
}

// Check if the output is empty or invalid JSON
if len(out) == 0 {
log.Printf("Trivy output is empty for image %s", image.PullableImage)
return // Move on to the next image
}

// Extract the JSON data from the output
var report model.Sbom
err = json.Unmarshal(out, &report)
if err != nil {
log.Printf("Error unmarshaling JSON data for image %s: %v", image.PullableImage, err)
return // Move on to the next image in case of an error
}

// Publish the report using the given function
publishTrivySbomReport(report, js, errCh)
}(i, image)
}

// Wait for all the goroutines to complete
wgc.Wait()
}
44 changes: 44 additions & 0 deletions client/pkg/clickhouse/db_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type DBInterface interface {
InsertKubeScoreMetrics(model.KubeScoreRecommendations)
InsertTrivyImageMetrics(metrics model.TrivyImage)
InsertTrivyMetrics(metrics model.Trivy)
InsertTrivySbomMetrics(metrics model.Reports)
RetriveKetallEvent() ([]model.Resource, error)
RetriveOutdatedEvent() ([]model.CheckResultfinal, error)
RetriveKubepugEvent() ([]model.Result, error)
Expand Down Expand Up @@ -278,6 +279,49 @@ func (c *DBClient) InsertKubvizEvent(metrics model.Metrics) {
log.Fatal(err)
}
}
func (c *DBClient) InsertTrivySbomMetrics(metrics model.Reports) {
result := metrics.Report
for _, com := range result.Components {
if len(result.Metadata.Tools) == 0 || len(com.Properties) == 0 || len(com.Hashes) == 0 || len(com.Licenses) == 0 {
continue
}
for _, depend := range result.Dependencies {
var (
tx, _ = c.conn.Begin()
stmt, _ = tx.Prepare(InsertTrivySbom)
)
if _, err := stmt.Exec(
metrics.ID,
result.Schema,
result.BomFormat,
result.SpecVersion,
result.SerialNumber,
result.Version,
result.Metadata.Timestamp,
result.Metadata.Tools[0].Vendor,
result.Metadata.Tools[0].Name,
result.Metadata.Tools[0].Version,
com.BomRef,
com.Type,
com.Name,
com.Version,
com.Properties[0].Name,
com.Properties[0].Value,
com.Hashes[0].Alg,
com.Hashes[0].Content,
com.Licenses[0].Expression,
com.Purl,
depend.Ref,
); err != nil {
log.Fatal(err)
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
stmt.Close()
}
}
}
func (c *DBClient) InsertGitEvent(event string) {
ctx := context.Background()
batch, err := c.splconn.PrepareBatch(ctx, "INSERT INTO git_json")
Expand Down
26 changes: 26 additions & 0 deletions client/pkg/clickhouse/statements.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,31 @@ const azureContainerPushEventTable DBStatement = `
SHAID String
) engine=File(TabSeparated)
`
const CreatetrivySbomTable DBStatement = `
CREATE TABLE IF NOT EXISTS trivysbom (
id UUID,
schema String,
bom_format String,
spec_version String,
serial_number String,
version INTEGER,
metadata_timestamp DateTime('UTC'),
metatool_vendor String,
metatool_name String,
metatool_version String,
component_bom_ref String,
component_type String,
component_name String,
component_version String,
component_property_name String,
component_property_value String,
component_hash_alg String,
component_hash_content String,
component_license_exp String,
component_purl String,
dependency_ref String
) engine=File(TabSeparated)
`

const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)"
const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)"
Expand All @@ -173,3 +198,4 @@ const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespac
const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"
const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
15 changes: 15 additions & 0 deletions client/pkg/clients/kubviz_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,21 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) {
log.Println()
},
},
{
Subject: constants.TRIVY_SBOM_SUBJECT,
Consumer: constants.Trivy_Sbom_Consumer,
Handler: func(msg *nats.Msg) {
msg.Ack()
var metrics model.Reports
err := json.Unmarshal(msg.Data, &metrics)
if err != nil {
log.Fatal(err)
}
log.Printf("Trivy sbom Metrics Received: %#v,", metrics)
conn.InsertTrivySbomMetrics(metrics)
log.Println()
},
},
{
Subject: constants.TRIVY_K8S_SUBJECT,
Consumer: constants.TrivyConsumer,
Expand Down
2 changes: 2 additions & 0 deletions constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,6 @@ const (
TrivyConsumer = "TRIVY_CONSUMER"
TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage"
Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER"
TRIVY_SBOM_SUBJECT = "METRICS.trivysbom"
Trivy_Sbom_Consumer = "TRIVY_SBOM_CONSUMER"
)
59 changes: 59 additions & 0 deletions model/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package model

import (
"time"
)

type Reports struct {
ID string
Report Sbom
}

type Sbom struct {
Schema string `json:"$schema"`
BomFormat string `json:"bomFormat"`
SpecVersion string `json:"specVersion"`
SerialNumber string `json:"serialNumber"`
Version int `json:"version"`
Metadata struct {
Timestamp time.Time `json:"timestamp"`
Tools []struct {
Vendor string `json:"vendor"`
Name string `json:"name"`
Version string `json:"version"`
} `json:"tools"`
Component struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Purl string `json:"purl"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
} `json:"component"`
} `json:"metadata"`
Components []struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Version string `json:"version"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
Hashes []struct {
Alg string `json:"alg"`
Content string `json:"content"`
} `json:"hashes,omitempty"`
Licenses []struct {
Expression string `json:"expression"`
} `json:"licenses,omitempty"`
Purl string `json:"purl,omitempty"`
} `json:"components"`
Dependencies []struct {
Ref string `json:"ref"`
DependsOn []string `json:"dependsOn"`
} `json:"dependencies"`
Vulnerabilities []interface{} `json:"vulnerabilities"`
}

0 comments on commit f1b7648

Please sign in to comment.