update trivy, and unsupport image scanning feature (future-architect#971

)

* update trivy, fanal. unsupport image scanning

* Update models/library.go

Co-authored-by: Teppei Fukuda <teppei@elab.ic.i.u-tokyo.ac.jp>

* add -no-progress flag to report/tui cmd

* Display trivy vuln info to tui/report

* add detection method to vulninfo detected by trivy

* fix(uuid): change uuid lib to go-uuid future-architect#929 (future-architect#969)

* update trivy, fanal. unsupport image scanning

* Update models/library.go

Co-authored-by: Teppei Fukuda <teppei@elab.ic.i.u-tokyo.ac.jp>

* add -no-progress flag to report/tui cmd

* Display trivy vuln info to tui/report

* add detection method to vulninfo detected by trivy

* unique ref links in TUI

* download trivy DB only when lock file is specified in config.toml

Co-authored-by: Teppei Fukuda <teppei@elab.ic.i.u-tokyo.ac.jp>

README.md

@@ -136,14 +136,16 @@ Vuls uses multiple vulnerability databases
 
 ### **Static** Analysis
 
-Vuls v0.8.0 can scan Docker images using [knqyf263/trivy](https://github.com/knqyf263/trivy).
-Following Registry supported.
+**Image scan function is no longer supported from Vuls v0.9.5. Use Trivy directry**
 
-- ECR
-- GCR
-- Local Image
+~~Vuls v0.8.0 can scan Docker images using [knqyf263/trivy](https://github.com/knqyf263/trivy).
+Following Registry supported.~~
 
-For details, see [Scan docker image](https://vuls.io/docs/en/tutorial-scan-docker-image.html)
+- ~~ECR~~
+- ~~GCR~~
+- ~~Local Image~~
+
+~~For details, see [Scan docker image](https://vuls.io/docs/en/tutorial-scan-docker-image.html)~~  
 
 ### Scan vulnerabilities of non-OS-packages
 

commands/report.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aquasecurity/trivy/pkg/utils"
 	c "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/exploit"
 	"github.com/future-architect/vuls/gost"
@@ -71,6 +72,7 @@ func (*ReportCmd) Usage() string {
 		[-debug]
 		[-debug-sql]
 		[-quiet]
+		[-no-progress]
 		[-pipe]
 		[-cvedb-type=sqlite3|mysql|postgres|redis|http]
 		[-cvedb-sqlite3-path=/path/to/cve.sqlite3]
@@ -85,6 +87,7 @@ func (*ReportCmd) Usage() string {
 		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
 		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
 		[-http="http://vuls-report-server"]
+		[-trivy-cachedb-dir=/path/to/dir]
 
 		[RFC3339 datetime format under results dir]
 `
@@ -95,8 +98,8 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&c.Conf.Lang, "lang", "en", "[en|ja]")
 	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
 	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "SQL debug mode")
-
 	f.BoolVar(&c.Conf.Quiet, "quiet", false, "Quiet mode. No output on stdout")
+	f.BoolVar(&c.Conf.NoProgress, "no-progress", false, "Suppress progress bar")
 
 	wd, _ := os.Getwd()
 	defaultConfPath := filepath.Join(wd, "config.toml")
@@ -187,6 +190,8 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 
 	f.StringVar(&p.httpConf.URL, "http", "", "-to-http http://vuls-report")
 
+	f.StringVar(&c.Conf.TrivyCacheDBDir, "trivy-cachedb-dir",
+		utils.DefaultCacheDir(), "/path/to/dir")
 }
 
 // Execute execute

commands/scan.go

@@ -41,7 +41,6 @@ func (*ScanCmd) Usage() string {
 		[-ssh-native-insecure]
 		[-ssh-config]
 		[-containers-only]
-		[-images-only]
 		[-libs-only]
 		[-wordpress-only]
 		[-skip-broken]
@@ -86,9 +85,6 @@ func (p *ScanCmd) SetFlags(f *flag.FlagSet) {
 	f.BoolVar(&c.Conf.ContainersOnly, "containers-only", false,
 		"Scan running containers only. Default: Scan both of hosts and running containers")
 
-	f.BoolVar(&c.Conf.ImagesOnly, "images-only", false,
-		"Scan container images only. Default: Scan both of hosts and images")
-
 	f.BoolVar(&c.Conf.LibsOnly, "libs-only", false,
 		"Scan libraries (lock files) specified in config.toml only.")
 

commands/tui.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/aquasecurity/trivy/pkg/utils"
 	c "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/exploit"
 	"github.com/future-architect/vuls/gost"
@@ -46,6 +47,8 @@ func (*TuiCmd) Usage() string {
 		[-log-dir=/path/to/log]
 		[-debug]
 		[-debug-sql]
+		[-quiet]
+		[-no-progress]
 		[-pipe]
 		[-cvedb-type=sqlite3|mysql|postgres|redis|http]
 		[-cvedb-sqlite3-path=/path/to/cve.sqlite3]
@@ -59,6 +62,7 @@ func (*TuiCmd) Usage() string {
 		[-exploitdb-type=sqlite3|mysql|redis|http]
 		[-exploitdb-sqlite3-path=/path/to/exploitdb.sqlite3]
 		[-exploitdb-url=http://127.0.0.1:1326 or DB connection string]
+		[-trivy-cachedb-dir=/path/to/dir]
 
 `
 }
@@ -68,6 +72,8 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
 	//  f.StringVar(&p.lang, "lang", "en", "[en|ja]")
 	f.BoolVar(&c.Conf.DebugSQL, "debug-sql", false, "debug SQL")
 	f.BoolVar(&c.Conf.Debug, "debug", false, "debug mode")
+	f.BoolVar(&c.Conf.Quiet, "quiet", false, "Quiet mode. No output on stdout")
+	f.BoolVar(&c.Conf.NoProgress, "no-progress", false, "Suppress progress bar")
 
 	defaultLogDir := util.GetDefaultLogDir()
 	f.StringVar(&c.Conf.LogDir, "log-dir", defaultLogDir, "/path/to/log")
@@ -121,6 +127,8 @@ func (p *TuiCmd) SetFlags(f *flag.FlagSet) {
 	f.StringVar(&p.exploitConf.URL, "exploitdb-url", "",
 		"http://exploit.com:1326 or DB connection string")
 
+	f.StringVar(&c.Conf.TrivyCacheDBDir, "trivy-cachedb-dir",
+		utils.DefaultCacheDir(), "/path/to/dir")
 }
 
 // Execute execute

config/config.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	syslog "github.com/RackSec/srslog"
-	"github.com/aquasecurity/fanal/types"
 	valid "github.com/asaskevich/govalidator"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/xerrors"
@@ -90,6 +89,7 @@ type Config struct {
 	ResultsDir string `json:"resultsDir,omitempty"`
 	Pipe       bool   `json:"pipe,omitempty"`
 	Quiet      bool   `json:"quiet,omitempty"`
+	NoProgress bool   `json:"noProgress,omitempty"`
 
 	Default       ServerInfo            `json:"default,omitempty"`
 	Servers       map[string]ServerInfo `json:"servers,omitempty"`
@@ -103,15 +103,16 @@ type Config struct {
 	SSHConfig bool `json:"sshConfig,omitempty"`
 
 	ContainersOnly bool `json:"containersOnly,omitempty"`
-	ImagesOnly     bool `json:"imagesOnly,omitempty"`
 	LibsOnly       bool `json:"libsOnly,omitempty"`
 	WordPressOnly  bool `json:"wordpressOnly,omitempty"`
 
-	SkipBroken  bool   `json:"skipBroken,omitempty"`
-	CacheDBPath string `json:"cacheDBPath,omitempty"`
-	Vvv         bool   `json:"vvv,omitempty"`
-	UUID        bool   `json:"uuid,omitempty"`
-	DetectIPS   bool   `json:"detectIps,omitempty"`
+	CacheDBPath     string `json:"cacheDBPath,omitempty"`
+	TrivyCacheDBDir string `json:"trivyCacheDBDir,omitempty"`
+
+	SkipBroken bool `json:"skipBroken,omitempty"`
+	Vvv        bool `json:"vvv,omitempty"`
+	UUID       bool `json:"uuid,omitempty"`
+	DetectIPS  bool `json:"detectIps,omitempty"`
 
 	CveDict  GoCveDictConf `json:"cveDict,omitempty"`
 	OvalDict GovalDictConf `json:"ovalDict,omitempty"`
@@ -1047,7 +1048,6 @@ type ServerInfo struct {
 	IgnoreCves             []string                    `toml:"ignoreCves,omitempty" json:"ignoreCves,omitempty"`
 	IgnorePkgsRegexp       []string                    `toml:"ignorePkgsRegexp,omitempty" json:"ignorePkgsRegexp,omitempty"`
 	GitHubRepos            map[string]GitHubConf       `toml:"githubs" json:"githubs,omitempty"` // key: owner/repo
-	Images                 map[string]Image            `toml:"images" json:"images,omitempty"`
 	UUIDs                  map[string]string           `toml:"uuids,omitempty" json:"uuids,omitempty"`
 	Memo                   string                      `toml:"memo,omitempty" json:"memo,omitempty"`
 	Enablerepo             []string                    `toml:"enablerepo,omitempty" json:"enablerepo,omitempty"` // For CentOS, RHEL, Amazon
@@ -1065,7 +1065,6 @@ type ServerInfo struct {
 
 	LogMsgAnsiColor string    `toml:"-" json:"-"` // DebugLog Color
 	Container       Container `toml:"-" json:"-"`
-	Image           Image     `toml:"-" json:"-"`
 	Distro          Distro    `toml:"-" json:"-"`
 	Mode            ScanMode  `toml:"-" json:"-"`
 }
@@ -1087,26 +1086,6 @@ type WordPressConf struct {
 	IgnoreInactive bool   `json:"ignoreInactive,omitempty"`
 }
 
-// Image is a scan container image info
-type Image struct {
-	Name             string             `json:"name"`
-	Tag              string             `json:"tag"`
-	Digest           string             `json:"digest"`
-	DockerOption     types.DockerOption `json:"dockerOption,omitempty"`
-	Cpes             []string           `json:"cpes,omitempty"`
-	OwaspDCXMLPath   string             `json:"owaspDCXMLPath"`
-	IgnorePkgsRegexp []string           `json:"ignorePkgsRegexp,omitempty"`
-	IgnoreCves       []string           `json:"ignoreCves,omitempty"`
-}
-
-// GetFullName returns a full name of the image
-func (i *Image) GetFullName() string {
-	if i.Digest != "" {
-		return i.Name + "@" + i.Digest
-	}
-	return i.Name + ":" + i.Tag
-}
-
 // GitHubConf is used for GitHub integration
 type GitHubConf struct {
 	Token string `json:"-"`

config/tomlloader.go

@@ -51,16 +51,6 @@ func (c TOMLLoader) Load(pathToToml, keyPass string) error {
 		}
 
 		s := ServerInfo{ServerName: serverName}
-		s.Images = make(map[string]Image)
-
-		// image are able to set any server type
-		for name, image := range v.Images {
-			if err := IsValidImage(image); err != nil {
-				return err
-			}
-			s.Images[name] = image
-		}
-
 		if v.Type != ServerTypePseudo {
 			s.Host = v.Host
 			if len(s.Host) == 0 {
@@ -292,17 +282,3 @@ func toCpeURI(cpename string) (string, error) {
 	}
 	return "", xerrors.Errorf("Unknow CPE format: %s", cpename)
 }
-
-// IsValidImage checks a container configuration
-func IsValidImage(c Image) error {
-	if c.Name == "" {
-		return xerrors.New("Invalid arguments : no image name")
-	}
-	if c.Tag == "" && c.Digest == "" {
-		return xerrors.New("Invalid arguments : no image tag and digest")
-	}
-	if c.Tag != "" && c.Digest != "" {
-		return xerrors.New("Invalid arguments : you can either set image tag or digest")
-	}
-	return nil
-}

config/tomlloader_test.go

@@ -42,62 +42,3 @@ func TestToCpeURI(t *testing.T) {
 		}
 	}
 }
-
-func TestIsValidImage(t *testing.T) {
-	var tests = []struct {
-		name     string
-		img      Image
-		errOccur bool
-	}{
-		{
-			name: "ok with tag",
-			img: Image{
-				Name: "ok",
-				Tag:  "ok",
-			},
-			errOccur: false,
-		},
-		{
-			name: "ok with digest",
-			img: Image{
-				Name:   "ok",
-				Digest: "ok",
-			},
-			errOccur: false,
-		},
-
-		{
-			name: "no image name with tag",
-			img: Image{
-				Tag: "ok",
-			},
-			errOccur: true,
-		},
-
-		{
-			name: "no image name with digest",
-			img: Image{
-				Digest: "ok",
-			},
-			errOccur: true,
-		},
-
-		{
-			name: "no tag and digest",
-			img: Image{
-				Name: "ok",
-			},
-			errOccur: true,
-		},
-	}
-	for i, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			err := IsValidImage(tt.img)
-			actual := err != nil
-			if actual != tt.errOccur {
-				t.Errorf("[%d] act: %v, exp: %v",
-					i, actual, tt.errOccur)
-			}
-		})
-	}
-}

go.mod

@@ -3,21 +3,20 @@ module github.com/future-architect/vuls
 go 1.13
 
 replace (
-	github.com/genuinetools/reg => github.com/tomoyamachi/reg v0.16.1-0.20190706172545-2a2250fd7c00
 	gopkg.in/mattn/go-colorable.v0 => github.com/mattn/go-colorable v0.1.0
 	gopkg.in/mattn/go-isatty.v0 => github.com/mattn/go-isatty v0.0.6
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go v41.2.0+incompatible
-	github.com/Azure/go-autorest/autorest v0.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go v42.0.0+incompatible
 	github.com/BurntSushi/toml v0.3.1
 	github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91
-	github.com/aquasecurity/fanal v0.0.0-20200124194549-91468b8e0460
-	github.com/aquasecurity/go-dep-parser v0.0.0-20190819075924-ea223f0ef24b
-	github.com/aquasecurity/trivy v0.1.6
+	github.com/aquasecurity/fanal v0.0.0-20200427221647-c3528846e21c
+	github.com/aquasecurity/go-dep-parser v0.0.0-20200123140603-4dc0125084da // indirect
+	github.com/aquasecurity/trivy v0.6.0
+	github.com/aquasecurity/trivy-db v0.0.0-20200427221211-19fb3b7a88b5
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a
-	github.com/aws/aws-sdk-go v1.30.9
+	github.com/aws/aws-sdk-go v1.30.16
 	github.com/boltdb/bolt v1.3.1
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/google/subcommands v1.2.0
@@ -44,9 +43,10 @@ require (
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/parnurzeal/gorequest v0.2.16
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
-	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.5.0
-	golang.org/x/crypto v0.0.0-20200414173820-0848c9571904
+	github.com/spf13/afero v1.2.2
+	golang.org/x/crypto v0.0.0-20200427165652-729f1e841bcc
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543
+	k8s.io/utils v0.0.0-20191114184206-e782cd3c129f
 )