Skip to content

Commit

Permalink
Merge pull request #179 from intelops/sbom-working
Browse files Browse the repository at this point in the history
sbom-fix
  • Loading branch information
avinashkna4 authored Aug 24, 2023
2 parents f6f3785 + d26339e commit a627bec
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 2 deletions.
9 changes: 8 additions & 1 deletion agent/kubviz/k8smetrics_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ func main() {
trivyK8sMetricsChan := make(chan error, 1)
kubescoreMetricsChan := make(chan error, 1)
trivyImagescanChan := make(chan error, 1)
trivySbomcanChan := make(chan error, 1)
RakeesErrChan := make(chan error, 1)
// Start a goroutine to handle errors
doneChan := make(chan bool)
Expand Down Expand Up @@ -133,6 +134,10 @@ func main() {
if err != nil {
log.Println(err)
}
case err := <-trivySbomcanChan:
if err != nil {
log.Println(err)
}
case err := <-trivyK8sMetricsChan:
if err != nil {
log.Println(err)
Expand All @@ -146,14 +151,15 @@ func main() {
}
}
}()
wg.Add(7) // Initialize the WaitGroup for the seven goroutines
wg.Add(8) // Initialize the WaitGroup for the seven goroutines
// ... start other goroutines ...
go outDatedImages(config, js, &wg, outdatedErrChan)
go KubePreUpgradeDetector(config, js, &wg, kubePreUpgradeChan)
go GetAllResources(config, js, &wg, getAllResourceChan)
go RakeesOutput(config, js, &wg, RakeesErrChan)
go getK8sEvents(clientset)
go RunTrivyImageScans(config, js, &wg, trivyImagescanChan)
go RunTrivySbomScan(config, js, &wg, trivySbomcanChan)
go RunKubeScore(clientset, js, &wg, kubescoreMetricsChan)
go RunTrivyK8sClusterScan(&wg, js, trivyK8sMetricsChan)
wg.Wait()
Expand All @@ -164,6 +170,7 @@ func main() {
// close(clusterMetricsChan)
close(kubescoreMetricsChan)
close(trivyImagescanChan)
close(trivySbomcanChan)
close(trivyK8sMetricsChan)
close(RakeesErrChan)
// Signal that all other goroutines have finished
Expand Down
84 changes: 84 additions & 0 deletions agent/kubviz/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package main

import (
"bytes"
"encoding/json"
"fmt"
"log"
"os/exec"
"sync"

"github.com/google/uuid"
"github.com/intelops/kubviz/constants"
"github.com/intelops/kubviz/model"
"github.com/nats-io/nats.go"
"k8s.io/client-go/rest"
)

func publishTrivySbomReport(report model.Sbom, js nats.JetStreamContext, errCh chan error) {
metrics := model.Reports{
ID: uuid.New().String(),
Report: report,
}
metricsJson, _ := json.Marshal(metrics)
_, err := js.Publish(constants.TRIVY_SBOM_SUBJECT, metricsJson)
if err != nil {
errCh <- err
}

log.Printf("Trivy report with BomFormat:%v has been published\n", metrics.Report.BomFormat)
errCh <- nil
}

func executeCommandSbom(command string) ([]byte, error) {
cmd := exec.Command("/bin/sh", "-c", command)
var outc, errc bytes.Buffer
cmd.Stdout = &outc
cmd.Stderr = &errc

err := cmd.Run()

if err != nil {
log.Println("Execute Command Error", err.Error())
}

return outc.Bytes(), err
}

func RunTrivySbomScan(config *rest.Config, js nats.JetStreamContext, wg *sync.WaitGroup, errCh chan error) {
log.Println("trivy sbom run started")
defer wg.Done()
images, err := ListImages(config)

if err != nil {
log.Printf("failed to list images: %v", err)
}
for _, image := range images {
fmt.Printf("pullable Image %#v\n", image.PullableImage)

command := fmt.Sprintf("trivy image --format cyclonedx %s %s", image.PullableImage, "--cache-dir /tmp/.cache")
out, err := executeCommandSbom(command)

if err != nil {
log.Printf("Error executing Trivy for image %s: %v", image.PullableImage, err)
continue // Move on to the next image in case of an error
}

// Check if the output is empty or invalid JSON
if len(out) == 0 {
log.Printf("Trivy output is empty for image %s", image.PullableImage)
continue // Move on to the next image
}

var report model.Sbom
err = json.Unmarshal(out, &report)
if err != nil {
log.Printf("Error unmarshaling JSON data for image %s: %v", image.PullableImage, err)
continue // Move on to the next image in case of an error
}
log.Println("report", report)

// Publish the report using the given function
publishTrivySbomReport(report, js, errCh)
}
}
53 changes: 52 additions & 1 deletion client/pkg/clickhouse/db_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ type DBInterface interface {
InsertGitEvent(string)
InsertKubeScoreMetrics(model.KubeScoreRecommendations)
InsertTrivyImageMetrics(metrics model.TrivyImage)
InsertTrivySbomMetrics(metrics model.Reports)
InsertTrivyMetrics(metrics model.Trivy)
RetriveKetallEvent() ([]model.Resource, error)
RetriveOutdatedEvent() ([]model.CheckResultfinal, error)
Expand Down Expand Up @@ -71,7 +72,7 @@ func NewDBClient(conf *config.Config) (DBInterface, error) {
return nil, err
}

tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
tables := []DBStatement{kubvizTable, rakeesTable, kubePugDepricatedTable, kubepugDeletedTable, ketallTable, trivyTableImage,trivySbomTable, outdateTable, clickhouseExperimental, containerDockerhubTable, containerGithubTable, kubescoreTable, trivyTableVul, trivyTableMisconfig, dockerHubBuildTable, azureContainerPushEventTable, DBStatement(dbstatement.AzureDevopsTable), DBStatement(dbstatement.GithubTable), DBStatement(dbstatement.GitlabTable), DBStatement(dbstatement.BitbucketTable), DBStatement(dbstatement.GiteaTable)}
for _, table := range tables {
if err = splconn.Exec(context.Background(), string(table)); err != nil {
return nil, err
Expand Down Expand Up @@ -431,6 +432,56 @@ func (c *DBClient) InsertTrivyImageMetrics(metrics model.TrivyImage) {

}
}
func (c *DBClient) InsertTrivySbomMetrics(metrics model.Reports) {
log.Println("####started inserting value")
result := metrics.Report
tx, err := c.conn.Begin()
if err != nil {
log.Println("error in conn Begin", err)
}
defer tx.Rollback()
stmt, err := tx.Prepare(InsertTrivySbom)
if err != nil {
log.Println("error in prepare", err)
}
defer stmt.Close()
for _, com := range result.Components {
if len(result.Metadata.Tools) == 0 || len(com.Properties) == 0 || len(com.Hashes) == 0 || len(com.Licenses) == 0 {
continue
}
for _, depend := range result.Dependencies {
if _, err := stmt.Exec(
metrics.ID,
result.Schema,
result.BomFormat,
result.SpecVersion,
result.SerialNumber,
int32(result.Version),
result.Metadata.Timestamp,
result.Metadata.Tools[0].Vendor,
result.Metadata.Tools[0].Name,
result.Metadata.Tools[0].Version,
com.BomRef,
com.Type,
com.Name,
com.Version,
com.Properties[0].Name,
com.Properties[0].Value,
com.Hashes[0].Alg,
com.Hashes[0].Content,
com.Licenses[0].Expression,
com.Purl,
depend.Ref,
); err != nil {
log.Fatal(err)
}
}
}
if err := tx.Commit(); err != nil {
log.Fatal(err)
}
log.Println("value inserted")
}
func (c *DBClient) Close() {
_ = c.conn.Close()
}
Expand Down
26 changes: 26 additions & 0 deletions client/pkg/clickhouse/statements.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,31 @@ const azureContainerPushEventTable DBStatement = `
SHAID String
) engine=File(TabSeparated)
`
const trivySbomTable DBStatement = `
CREATE TABLE IF NOT EXISTS trivysbom (
id UUID,
schema String,
bom_format String,
spec_version String,
serial_number String,
version INTEGER,
metadata_timestamp DateTime('UTC'),
metatool_vendor String,
metatool_name String,
metatool_version String,
component_bom_ref String,
component_type String,
component_name String,
component_version String,
component_property_name String,
component_property_value String,
component_hash_alg String,
component_hash_content String,
component_license_exp String,
component_purl String,
dependency_ref String
) engine=File(TabSeparated)
`

const InsertDockerHubBuild DBStatement = "INSERT INTO dockerhubbuild (PushedBy, ImageTag, RepositoryName, DateCreated, Owner, Event) VALUES (?, ?, ?, ?, ?, ?)"
const InsertRakees DBStatement = "INSERT INTO rakkess (ClusterName, Name, Create, Delete, List, Update) VALUES (?, ?, ?, ?, ?, ?)"
Expand All @@ -173,3 +198,4 @@ const InsertTrivyVul string = "INSERT INTO trivy_vul (id, cluster_name, namespac
const InsertTrivyImage string = "INSERT INTO trivyimage (id, cluster_name, artifact_name, vul_id, vul_pkg_id, vul_pkg_name, vul_installed_version, vul_fixed_version, vul_title, vul_severity, vul_published_date, vul_last_modified_date) VALUES ( ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivyMisconfig string = "INSERT INTO trivy_misconfig (id, cluster_name, namespace, kind, name, misconfig_id, misconfig_avdid, misconfig_type, misconfig_title, misconfig_desc, misconfig_msg, misconfig_query, misconfig_resolution, misconfig_severity, misconfig_status) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?. ?, ?)"
const InsertAzureContainerPushEvent DBStatement = "INSERT INTO azurecontainerpush (RegistryURL, RepositoryName, Tag, ImageName, Event, Timestamp, Size, SHAID) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
const InsertTrivySbom string = "INSERT INTO trivysbom (id, schema, bom_format,spec_version,serial_number, version, metadata_timestamp,metatool_vendor,metatool_name,metatool_version,component_bom_ref,component_type,component_name,component_version,component_property_name,component_property_value,component_hash_alg,component_hash_content,component_license_exp,component_purl,dependency_ref) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
15 changes: 15 additions & 0 deletions client/pkg/clients/kubviz_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,21 @@ func (n *NATSContext) SubscribeAllKubvizNats(conn clickhouse.DBInterface) {
log.Println()
},
},
{
Subject: constants.TRIVY_SBOM_SUBJECT,
Consumer: constants.Trivy_Sbom_Consumer,
Handler: func(msg *nats.Msg) {
msg.Ack()
var metrics model.Reports
err := json.Unmarshal(msg.Data, &metrics)
if err != nil {
log.Println("failed to unmarshal in nats", err)
}
log.Printf("Trivy sbom Metrics Received: %#v,", metrics)
conn.InsertTrivySbomMetrics(metrics)
log.Println()
},
},
{
Subject: constants.KubvizSubject,
Consumer: constants.KubvizConsumer,
Expand Down
2 changes: 2 additions & 0 deletions constants/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,6 @@ const (
TrivyConsumer = "TRIVY_CONSUMER"
TRIVY_IMAGE_SUBJECT = "METRICS.trivyimage"
Trivy_Image_Consumer = "TRIVY_IMAGE_CONSUMER"
TRIVY_SBOM_SUBJECT = "METRICS.trivysbom"
Trivy_Sbom_Consumer = "TRIVY_SBOM_CONSUMER"
)
59 changes: 59 additions & 0 deletions model/trivy_sbom.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package model

import (
"time"
)

type Reports struct {
ID string
Report Sbom
}

type Sbom struct {
Schema string `json:"$schema"`
BomFormat string `json:"bomFormat"`
SpecVersion string `json:"specVersion"`
SerialNumber string `json:"serialNumber"`
Version int `json:"version"`
Metadata struct {
Timestamp time.Time `json:"timestamp"`
Tools []struct {
Vendor string `json:"vendor"`
Name string `json:"name"`
Version string `json:"version"`
} `json:"tools"`
Component struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Purl string `json:"purl"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
} `json:"component"`
} `json:"metadata"`
Components []struct {
BomRef string `json:"bom-ref"`
Type string `json:"type"`
Name string `json:"name"`
Version string `json:"version"`
Properties []struct {
Name string `json:"name"`
Value string `json:"value"`
} `json:"properties"`
Hashes []struct {
Alg string `json:"alg"`
Content string `json:"content"`
} `json:"hashes,omitempty"`
Licenses []struct {
Expression string `json:"expression"`
} `json:"licenses,omitempty"`
Purl string `json:"purl,omitempty"`
} `json:"components"`
Dependencies []struct {
Ref string `json:"ref"`
DependsOn []string `json:"dependsOn"`
} `json:"dependencies"`
Vulnerabilities []interface{} `json:"vulnerabilities"`
}

0 comments on commit a627bec

Please sign in to comment.