fix(cve): Fix CVE scanning in images containing Jar files

Makefile

@@ -117,8 +117,14 @@ privileged-test: check-skopeo $(TESTDATA)
 	go test -failfast -tags needprivileges,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/... ./pkg/cli/... -run ^TestElevatedPrivileges
 
 $(TESTDATA): check-skopeo
-	$(shell mkdir -p ${TESTDATA}; cd ${TESTDATA}; mkdir -p noidentity; ../scripts/gen_certs.sh; cd ${TESTDATA}/noidentity; ../../scripts/gen_nameless_certs.sh; cd ${TOP_LEVEL}; skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1;skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1)
-	$(shell chmod -R a=rwx ${TESTDATA})
+	mkdir -p ${TESTDATA}; \
+	cd ${TESTDATA}; ../scripts/gen_certs.sh; \
+	mkdir -p noidentity; cd ${TESTDATA}/noidentity; ../../scripts/gen_nameless_certs.sh; \
+	cd ${TOP_LEVEL}; \
+	skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1; \
+	skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1; \
+	skopeo --insecure-policy copy -q docker://ghcr.io/project-zot/test-images/java:0.0.1 oci:${TESTDATA}/zot-cve-java-test:0.0.1; \
+	chmod -R a=rwx ${TESTDATA}
 
 .PHONY: run-bench
 run-bench: binary bench
@@ -242,6 +248,7 @@ clean:
 	rm -rf hack
 	rm -rf test/data/zot-test
 	rm -rf test/data/zot-cve-test
+	rm -rf test/data/zot-cve-java-test
 	rm -rf pkg/extensions/build
 
 .PHONY: run

go.mod

@@ -58,6 +58,7 @@ require (
 	github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230117141039-067a0f5b0e25
 	github.com/sigstore/cosign/v2 v2.0.2
 	github.com/swaggo/http-swagger v1.3.4
+	modernc.org/sqlite v1.20.3
 	oras.land/oras-go/v2 v2.2.0
 )
 
@@ -111,6 +112,7 @@ require (
 	github.com/gosuri/uitable v0.0.4 // indirect
 	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/liamg/iamgo v0.0.9 // indirect
@@ -142,6 +144,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect
 	github.com/rubenv/sql-migrate v1.2.0 // indirect
 	github.com/samber/lo v1.37.0 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
@@ -164,6 +167,15 @@ require (
 	k8s.io/cli-runtime v0.26.3 // indirect
 	k8s.io/component-base v0.26.3 // indirect
 	k8s.io/kubectl v0.26.3 // indirect
+	lukechampine.com/uint128 v1.2.0 // indirect
+	modernc.org/cc/v3 v3.40.0 // indirect
+	modernc.org/ccgo/v3 v3.16.13 // indirect
+	modernc.org/libc v1.22.2 // indirect
+	modernc.org/mathutil v1.5.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
+	modernc.org/opt v0.1.3 // indirect
+	modernc.org/strutil v1.1.3 // indirect
+	modernc.org/token v1.0.1 // indirect
 	oras.land/oras-go v1.2.3 // indirect
 	sigs.k8s.io/kustomize/api v0.12.1 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect

go.sum

@@ -937,7 +937,7 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe
 github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
 github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
-github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY=
+github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/s2a-go v0.1.3 h1:FAgZmpLl/SXurPEZyCMPBIiiYeTbqfjlbdnCNTAkbGE=
 github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
@@ -1094,6 +1094,7 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh
 github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
+github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -1416,7 +1417,9 @@ github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RN
 github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b/go.mod h1:KjY0wibdYKc4DYkerHSbguaf3JeIPGhNJBp2BNiFH78=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M=
+github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
 github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
@@ -2431,15 +2434,29 @@ k8s.io/kubectl v0.26.3/go.mod h1:02+gv7Qn4dupzN3fi/9OvqqdW+uG/4Zi56vc4Zmsp1g=
 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk=
 k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
+lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
+modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
 modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
+modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
+modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
+modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
 modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0=
+modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
 modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
+modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
 modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
+modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
 modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
+modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs=
+modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
 modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
+modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
+modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34=
 modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
+modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE=
 oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=
 oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg=
 oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=

pkg/cli/extensions_test.go

@@ -641,6 +641,13 @@ func TestServeSearchEnabled(t *testing.T) {
 		substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}`
 
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+
+		if !found {
+			data, err := os.ReadFile(logPath)
+			So(err, ShouldBeNil)
+			t.Log(string(data))
+		}
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 	})
@@ -680,18 +687,24 @@ func TestServeSearchEnabledCVE(t *testing.T) {
 		// to avoid data race when multiple go routines write to trivy DB instance.
 		WaitTillTrivyDBDownloadStarted(tempDir)
 
-		substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":null}}"
+		// The default config handling logic will convert the 1h interval to a 2h interval
+		substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":7200000000000,\"Trivy\":" +
+			"{\"DBRepository\":\"ghcr.io/aquasecurity/trivy-db\",\"JavaDBRepository\":\"ghcr.io/aquasecurity/trivy-java-db\"}}}"
 
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
-		So(found, ShouldBeTrue)
-		So(err, ShouldBeNil)
 
-		found, err = readLogFileAndSearchString(logPath, "updating the CVE database", readLogFileTimeout)
+		defer func() {
+			if !found {
+				data, err := os.ReadFile(logPath)
+				So(err, ShouldBeNil)
+				t.Log(string(data))
+			}
+		}()
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 
-		substring = "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing." //nolint:lll // gofumpt conflicts with lll
-		found, err = readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+		found, err = readLogFileAndSearchString(logPath, "updating the CVE database", readLogFileTimeout)
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 	})
@@ -729,6 +742,13 @@ func TestServeSearchEnabledNoCVE(t *testing.T) {
 
 		substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}` //nolint:lll // gofumpt conflicts with lll
 		found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
+
+		if !found {
+			data, err := os.ReadFile(logPath)
+			So(err, ShouldBeNil)
+			t.Log(string(data))
+		}
+
 		So(found, ShouldBeTrue)
 		So(err, ShouldBeNil)
 	})

pkg/cli/root.go

@@ -448,7 +448,7 @@ func validateAuthzPolicies(config *config.Config) error {
 	return nil
 }
 
-//nolint:gocyclo
+//nolint:gocyclo,cyclop,nestif
 func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
 	defaultVal := true
 
@@ -503,6 +503,35 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
 			if config.Extensions.Search.Enable == nil {
 				config.Extensions.Search.Enable = &defaultVal
 			}
+
+			if *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
+				defaultUpdateInterval, _ := time.ParseDuration("2h")
+
+				if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval {
+					config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval
+
+					log.Warn().Msg("CVE update interval set to too-short interval < 2h, " +
+						"changing update duration to 2 hours and continuing.")
+				}
+
+				if config.Extensions.Search.CVE.Trivy == nil {
+					config.Extensions.Search.CVE.Trivy = &extconf.TrivyConfig{}
+				}
+
+				if config.Extensions.Search.CVE.Trivy.DBRepository == "" {
+					defaultDBDownloadURL := "ghcr.io/aquasecurity/trivy-db"
+					log.Info().Str("trivyDownloadURL", defaultDBDownloadURL).
+						Msg("Config: using default Trivy DB download URL.")
+					config.Extensions.Search.CVE.Trivy.DBRepository = defaultDBDownloadURL
+				}
+
+				if config.Extensions.Search.CVE.Trivy.JavaDBRepository == "" {
+					defaultJavaDBDownloadURL := "ghcr.io/aquasecurity/trivy-java-db"
+					log.Info().Str("trivyJavaDownloadURL", defaultJavaDBDownloadURL).
+						Msg("Config: using default Trivy Java DB download URL.")
+					config.Extensions.Search.CVE.Trivy.JavaDBRepository = defaultJavaDBDownloadURL
+				}
+			}
 		}
 
 		if config.Extensions.Metrics != nil {

pkg/extensions/config/config.go

@@ -42,7 +42,8 @@ type CVEConfig struct {
 }
 
 type TrivyConfig struct {
-	DBRepository string // default is "ghcr.io/aquasecurity/trivy-db"
+	DBRepository     string // default is "ghcr.io/aquasecurity/trivy-db"
+	JavaDBRepository string // default is "ghcr.io/aquasecurity/trivy-java-db"
 }
 
 type MetricsConfig struct {

pkg/extensions/extension_search.go

@@ -45,27 +45,16 @@ func GetCVEInfo(config *config.Config, storeController storage.StoreController,
 		return nil
 	}
 
-	dbRepository := ""
+	dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository
+	javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository
 
-	if config.Extensions.Search.CVE.Trivy != nil {
-		dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository
-	}
-
-	return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log)
+	return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, javaDBRepository, log)
 }
 
 func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
 	repoDB repodb.RepoDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
 ) {
 	if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
-		defaultUpdateInterval, _ := time.ParseDuration("2h")
-
-		if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval {
-			config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval
-
-			log.Warn().Msg("CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
-		}
-
 		updateInterval := config.Extensions.Search.CVE.UpdateInterval
 
 		downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log)
@@ -77,6 +66,7 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC
 func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) {
 	generator := NewTrivyTaskGenerator(interval, cveInfo, log)
 
+	log.Info().Msg("Submitting CVE DB update scheduler")
 	sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
 }
 

pkg/extensions/extension_search_test.go

@@ -46,7 +46,7 @@ func TestTrivyDBGenerator(t *testing.T) {
 			DefaultStore: mocks.MockedImageStore{},
 		}
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", logger)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", logger)
 		generator := NewTrivyTaskGenerator(time.Minute, cveInfo, logger)
 
 		sch.SubmitGenerator(generator, 12000*time.Millisecond, scheduler.HighPriority)

pkg/extensions/search/cve/cve.go

@@ -46,9 +46,9 @@ type BaseCveInfo struct {
 }
 
 func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB,
-	dbRepository string, log log.Logger,
+	dbRepository, javaDBRepository string, log log.Logger,
 ) *BaseCveInfo {
-	scanner := trivy.NewScanner(storeController, repoDB, dbRepository, log)
+	scanner := trivy.NewScanner(storeController, repoDB, dbRepository, javaDBRepository, log)
 
 	return &BaseCveInfo{
 		Log:     log,

pkg/extensions/search/cve/cve_test.go

@@ -327,7 +327,7 @@ func TestImageFormat(t *testing.T) {
 		err = repodb.ParseStorage(repoDB, storeController, log)
 		So(err, ShouldBeNil)
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
 		isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test", "")
 		So(err, ShouldNotBeNil)
@@ -390,7 +390,7 @@ func TestImageFormat(t *testing.T) {
 			DefaultStore: mocks.MockedImageStore{},
 		}
 
-		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
+		cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log)
 
 		isScanable, err := cveInfo.Scanner.IsImageFormatScannable("repo", "tag")
 		So(err, ShouldBeNil)