Merge pull request #1667 from cpanato/vulndash-logs

vulndash: Add page size when listing vulnerabilities more logs and build v0.3.0 image

cmd/vulndash/Makefile

@@ -20,7 +20,7 @@ SHELL=/bin/bash -o pipefail
 
 REGISTRY ?= gcr.io/k8s-staging-artifact-promoter
 IMGNAME = vulndash
-IMAGE_VERSION ?= v0.2.1
+IMAGE_VERSION ?= v0.3.0
 CONFIG ?= buster
 
 IMAGE = $(REGISTRY)/$(IMGNAME)

cmd/vulndash/cmd/root.go

@@ -48,6 +48,7 @@ type options struct {
 	project           string
 	bucket            string
 	dashboardFilePath string
+	pageSize          int32
 	logLevel          string
 }
 
@@ -57,6 +58,7 @@ var (
 	projectFlag           = "project"
 	bucketFlag            = "bucket"
 	dashboardFilePathFlag = "dashboard-file-path"
+	pageSizeFlag          = "page-size"
 
 	// requiredFlags only if the config flag is not set
 	requiredFlags = []string{
@@ -102,6 +104,13 @@ func init() {
 		"info",
 		"the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'",
 	)
+
+	rootCmd.PersistentFlags().Int32Var(
+		&opts.pageSize,
+		pageSizeFlag,
+		200,
+		"the page size when getting the list of vulnerabilities",
+	)
 }
 
 func initLogging(*cobra.Command, []string) error {
@@ -132,6 +141,7 @@ func run(opts *options) error {
 		opts.dashboardFilePath,
 		opts.project,
 		opts.bucket,
+		opts.pageSize,
 	)
 	if updateErr != nil {
 		return errors.Wrap(updateErr, "updating vulnerability dashboard")

cmd/vulndash/variants.yaml

@@ -1,5 +1,5 @@
 variants:
   default:
-    IMAGE_VERSION: 'v0.2.1'
+    IMAGE_VERSION: 'v0.3.0'
     GO_VERSION: '1.15.3'
     DISTROLESS_IMAGE: 'static-debian10'

dependencies.yaml

@@ -120,7 +120,7 @@ dependencies:
 
   # Images: k8s.io/artifact-promoter
   - name: "k8s.io/artifact-promoter/vulndash"
-    version: v0.2.1
+    version: v0.3.0
     refPaths:
     - path: cmd/vulndash/Makefile
       match: IMAGE_VERSION\ \?=\ v((([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?)

pkg/vulndash/adapter/adapter.go

@@ -30,6 +30,7 @@ import (
 	containeranalysis "cloud.google.com/go/containeranalysis/apiv1"
 	"cloud.google.com/go/storage"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/html"
 	"google.golang.org/api/iterator"
 	grafeaspb "google.golang.org/genproto/googleapis/grafeas/v1"
@@ -71,6 +72,7 @@ func uploadFile(directory, filename, bucket string) error {
 // with images in a specific project using the Container Analysis Service.
 func GetAllVulnerabilities(
 	projectID string,
+	pageSize int32,
 ) ([]*grafeaspb.Occurrence, error) {
 	ctx := context.Background()
 	client, err := containeranalysis.NewClient(ctx)
@@ -80,14 +82,18 @@ func GetAllVulnerabilities(
 	defer client.Close()
 
 	req := &grafeaspb.ListOccurrencesRequest{
-		Parent: fmt.Sprintf("projects/%s", projectID),
-		Filter: fmt.Sprintf("kind = %q", "VULNERABILITY"),
+		Parent:   fmt.Sprintf("projects/%s", projectID),
+		Filter:   fmt.Sprintf("kind = %q", "VULNERABILITY"),
+		PageSize: pageSize,
 	}
 
+	logrus.Info("listing the vulnerabilities, will take a while...")
 	var occurrenceList []*grafeaspb.Occurrence
 	it := client.GetGrafeasClient().ListOccurrences(ctx, req)
 	for {
-		occ, err := it.Next()
+		var occ *grafeaspb.Occurrence
+		var err error
+		occ, err = it.Next()
 		if err == iterator.Done {
 			break
 		}
@@ -96,6 +102,7 @@ func GetAllVulnerabilities(
 		}
 		occurrenceList = append(occurrenceList, occ)
 	}
+	logrus.Infof("done listing the vulnerabilities")
 
 	return occurrenceList, err
 }
@@ -164,46 +171,57 @@ func UpdateVulnerabilityDashboard(
 	dashboardPath string,
 	vulnProject string,
 	dashboardBucket string,
+	pageSize int32,
 ) error {
-	htmlReader, openErr := os.Open(dashboardPath + "dashboard.html")
+	dashboardHTML := dashboardPath + "dashboard.html"
+	logrus.Infof("opening %s", dashboardHTML)
+	htmlReader, openErr := os.Open(dashboardHTML)
 	if openErr != nil {
 		return errors.Wrap(openErr, "opening dashboard file")
 	}
 
+	logrus.Infof("parsing %s", dashboardHTML)
 	_, err := html.Parse(htmlReader)
 	if err != nil {
 		return errors.Errorf("dashboard.html is not valid HTML: %v", err)
 	}
+
+	logrus.Infof("uploading %s to gcs", dashboardHTML)
 	err = uploadFile(dashboardPath, "dashboard.html", dashboardBucket)
 	if err != nil {
 		return errors.Errorf("Unable to upload latest version of "+
 			"dashboard HTML: %v", err)
 	}
 
+	logrus.Info("uploading updated dashboard.js to gcs")
 	err = uploadFile(dashboardPath, "dashboard.js", dashboardBucket)
 	if err != nil {
 		return errors.Errorf("Unable to upload latest version of "+
 			"dashboard JS: %v", err)
 	}
 
-	productionVulnerabilities, getVulnErr := GetAllVulnerabilities(vulnProject)
+	logrus.Infof("checking all vulnerabilities for %s", vulnProject)
+	productionVulnerabilities, getVulnErr := GetAllVulnerabilities(vulnProject, pageSize)
 	if getVulnErr != nil {
 		return errors.Wrap(getVulnErr, "getting all vulnerabilities")
 	}
 
+	logrus.Infof("parsing the vulnerabilities for %s", vulnProject)
 	vulnBreakdowns := GenerateVulnerabilityBreakdown(productionVulnerabilities)
 	jsonFile, err := json.MarshalIndent(vulnBreakdowns, "", " ")
 	if err != nil {
 		return errors.Errorf("Unable to generate dashboard json: %v", err)
 	}
 
-	err = ioutil.WriteFile(dashboardPath+"dashboard.json",
-		jsonFile, 0644)
+	dashboardJSON := dashboardPath + "dashboard.json"
+	logrus.Infof("writing the vulnerabilities for %s in the file %s", vulnProject, dashboardJSON)
+	err = ioutil.WriteFile(dashboardJSON, jsonFile, 0644)
 	if err != nil {
 		return errors.Errorf("Unable to create temporary local"+
 			"JSON file for the dashboard: %v", err)
 	}
 
+	logrus.Infof("uploading updated %s to gcs", dashboardJSON)
 	err = uploadFile(dashboardPath, "dashboard.json", dashboardBucket)
 	if err != nil {
 		return errors.Errorf("Unable to upload latest version of "+