Skip to content

ROX-12577 Scanner: load Istio dump #955

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 17, 2022
Merged

ROX-12577 Scanner: load Istio dump #955

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
6927f28
First commit
daynewlee Sep 30, 2022
1ad2b56
Minor update
daynewlee Sep 30, 2022
29dea70
Fixed istio cve path
daynewlee Sep 30, 2022
f0c41b1
Added istio dump to genesis and Added zip file step and Updated image…
daynewlee Oct 1, 2022
cab33bb
Merge branch 'master' into personal/yli3/addIstioDump
daynewlee Oct 10, 2022
f8fd461
Minor fix on e2e test and Added generateIstioDiffHelper and Added typ…
daynewlee Oct 10, 2022
406c7b5
Minor fix based on comments
daynewlee Oct 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .openshift-ci/build/build-bundle.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,13 @@ get_genesis_dump() {
mkdir -p /tmp/vuln-dump
gsutil cp gs://stackrox-scanner-ci-vuln-dump/nvd-definitions.zip /tmp/vuln-dump/nvd-definitions.zip
gsutil cp gs://stackrox-scanner-ci-vuln-dump/k8s-definitions.zip /tmp/vuln-dump/k8s-definitions.zip
gsutil cp gs://stackrox-scanner-ci-vuln-dump/istio-definitions.zip /tmp/vuln-dump/istio-definitions.zip
gsutil cp gs://stackrox-scanner-ci-vuln-dump/repo2cpe.zip /tmp/vuln-dump/repo2cpe.zip
fi

unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/nvd-definitions.zip
unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/k8s-definitions.zip
unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/istio-definitions.zip
unzip -d "$ROOT/image/scanner/dump" /tmp/vuln-dump/repo2cpe.zip
}

Expand Down
1 change: 1 addition & 0 deletions .openshift-ci/build/generate-genesis-dump.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ generate_genesis_dump() {
mkdir -p /tmp/vuln-dump
zip /tmp/genesis-dump/genesis-dump.zip 'nvd/*' --copy --out /tmp/vuln-dump/nvd-definitions.zip
zip /tmp/genesis-dump/genesis-dump.zip 'k8s/*' --copy --out /tmp/vuln-dump/k8s-definitions.zip
zip /tmp/genesis-dump/genesis-dump.zip 'istio/*' --copy --out /tmp/vuln-dump/istio-definitions.zip
zip /tmp/genesis-dump/genesis-dump.zip 'rhelv2/repository-to-cpe.json' --copy --out /tmp/vuln-dump/repo2cpe.zip
}

Expand Down
79 changes: 71 additions & 8 deletions cmd/updater/diffdumps/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/stackrox/istio-cves/types"
"github.com/stackrox/k8s-cves/pkg/validation"
"github.com/stackrox/rox/pkg/utils"
"github.com/stackrox/scanner/cmd/updater/common"
Expand All @@ -24,11 +25,14 @@ import (
"github.com/stackrox/scanner/ext/vulnsrc/alpine"
"github.com/stackrox/scanner/ext/vulnsrc/ubuntu"
"github.com/stackrox/scanner/pkg/vulndump"
"github.com/stackrox/scanner/pkg/vulnloader/istioloader"
"github.com/stackrox/scanner/pkg/vulnloader/k8sloader"
"github.com/stackrox/scanner/pkg/vulnloader/nvdloader"
namespaces "github.com/stackrox/scanner/pkg/wellknownnamespaces"
)

type generateDiffFunc func(outputDir string, baseF, headF *zip.File) error

func generateK8sDiff(outputDir string, baseF, headF *zip.File) error {
headReader, err := headF.Open()
if err != nil {
Expand Down Expand Up @@ -69,9 +73,57 @@ func generateK8sDiff(outputDir string, baseF, headF *zip.File) error {
return nil
}

func generateIstioDiff(outputDir string, baseF, headF *zip.File) error {
headReader, err := headF.Open()
if err != nil {
return errors.Wrap(err, "opening file")
}
defer utils.IgnoreError(headReader.Close)
istioDump, err := istioloader.LoadYAMLFileFromReader(headReader)
if err != nil {
return errors.Wrap(err, "reading Istio dump")
}

var baseIstioDump types.Vuln
if baseF != nil {
baseReader, err := baseF.Open()
if err != nil {
return errors.Wrap(err, "opening file")
}
defer utils.IgnoreError(baseReader.Close)
baseIstioDump, err = istioloader.LoadYAMLFileFromReader(baseReader)
if err != nil {
return errors.Wrap(err, "reading base Istio dump")
}
}

outF, err := os.Create(filepath.Join(outputDir, filepath.Base(headF.Name)))
if err != nil {
return errors.Wrap(err, "creating Istio output file")
}
defer utils.IgnoreError(outF.Close)

if !reflect.DeepEqual(baseIstioDump, istioDump) {
log.Infof("Istio CVE file %q is in the diff", headF.Name)
if _, err := io.Copy(outF, headReader); err != nil {
return errors.Wrap(err, "copying Istio CVE file")
}
}

return nil
}

func generateK8sDiffs(outputDir string, baseZipR *zip.ReadCloser, headZipR *zip.ReadCloser) error {
k8sSubDir := filepath.Join(outputDir, vulndump.K8sDirName)
if err := os.MkdirAll(k8sSubDir, 0755); err != nil {
return generateDiffsHelper(outputDir, baseZipR, headZipR, vulndump.K8sDirName, generateK8sDiff)
}

func generateIstioDiffs(outputDir string, baseZipR *zip.ReadCloser, headZipR *zip.ReadCloser) error {
return generateDiffsHelper(outputDir, baseZipR, headZipR, vulndump.IstioDirName, generateIstioDiff)
}

func generateDiffsHelper(outputDir string, baseZipR *zip.ReadCloser, headZipR *zip.ReadCloser, dirName string, generateDiffs generateDiffFunc) error {
subDir := filepath.Join(outputDir, dirName)
if err := os.MkdirAll(subDir, 0755); err != nil {
return errors.Wrap(err, "creating subdir for Kubernetes")
}

Expand All @@ -84,7 +136,7 @@ func generateK8sDiffs(outputDir string, baseZipR *zip.ReadCloser, headZipR *zip.
continue
}

if filepath.Dir(name) == vulndump.K8sDirName && filepath.Ext(name) == ".yaml" {
if filepath.Dir(name) == dirName && filepath.Ext(name) == ".yaml" {
baseFiles[name] = baseF
}
}
Expand All @@ -98,12 +150,12 @@ func generateK8sDiffs(outputDir string, baseZipR *zip.ReadCloser, headZipR *zip.
continue
}

// Only look at YAML files in the k8s/ folder.
if filepath.Dir(name) != vulndump.K8sDirName || filepath.Ext(name) != ".yaml" {
// Only look at YAML files in the <dirName> folder.
if filepath.Dir(name) != dirName || filepath.Ext(name) != ".yaml" {
continue
}
if err := generateK8sDiff(k8sSubDir, baseFiles[name], headF); err != nil {
return errors.Wrapf(err, "generating Kubernetes diff for file %q", headF.Name)
if err := generateDiffs(subDir, baseFiles[name], headF); err != nil {
return errors.Wrapf(err, "generating diffs for file %q", headF.Name)
}
}
return nil
Expand Down Expand Up @@ -343,6 +395,7 @@ func generateOSVulnsDiff(outputDir string, baseZipR, headZipR *zip.ReadCloser, c
type config struct {
SkipFixableCentOSVulns bool `json:"skipFixableCentOSVulns"`
IgnoreKubernetesVulns bool `json:"ignoreKubernetesVulns"`
IgnoreIstioVulns bool `json:"ignoreIstioVulns"`
SkipUbuntuLinuxKernelVulns bool `json:"skipUbuntuLinuxKernelVulns"`
SkipSeverityComparison bool `json:"skipSeverityComparison"`
SkipRHELv2Vulns bool `json:"skipRHELv2Vulns"`
Expand Down Expand Up @@ -409,6 +462,16 @@ func Command() *cobra.Command {
log.Info("Done generating Kubernetes diff")
}

if cfg.IgnoreIstioVulns {
log.Info("Skipping Istio diff")
} else {
log.Info("Generating Istio diff...")
if err := generateIstioDiffs(stagingDir, baseZipR, headZipR); err != nil {
return errors.Wrap(err, "creating Istio diff")
}
log.Info("Done generating Isio diff")
}

log.Info("Generating NVD diff...")
if err := generateNVDDiffs(stagingDir, baseManifest.Until, headZipR); err != nil {
return errors.Wrap(err, "creating NVD diff")
Expand Down Expand Up @@ -440,7 +503,7 @@ func Command() *cobra.Command {
}

log.Info("Zipping up the dump...")
err = vulndump.WriteZip(stagingDir, outFile, cfg.IgnoreKubernetesVulns, cfg.SkipRHELv2Vulns)
err = vulndump.WriteZip(stagingDir, outFile, cfg.IgnoreKubernetesVulns, cfg.SkipRHELv2Vulns, cfg.IgnoreIstioVulns)
if err != nil {
return errors.Wrap(err, "writing final zip")
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/updater/generatedump/cmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func generateDumpWithAllVulns(outFile string) error {
}

log.Info("Zipping up the files...")
if err := vulndump.WriteZip(dumpDir, outFile, false, false); err != nil {
if err := vulndump.WriteZip(dumpDir, outFile, false, false, false); err != nil {
return errors.Wrap(err, "creating ZIP of the vuln dump")
}
log.Info("Done writing the zip with the entire vuln dump!")
Expand Down
2 changes: 1 addition & 1 deletion e2etests/testcase_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1990,7 +1990,7 @@ var testCases = []testCase{
},
AddedBy: "sha256:36e8e9714b9a509fae9e515ff16237928c3d809f5ae228b14d2f7d7605c02623",
Location: "jars/jackson-databind-2.9.10.4.jar",
FixedBy: "2.12.6.1",
FixedBy: "2.14.0",
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should really be done in a separate PR, but it's fine for now

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I was using another pr but the CI always timed out. Meanwhile I didn't remove this change from this pr.

},
},
unexpectedFeatures: []apiV1.Feature{
Expand Down
1 change: 1 addition & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,7 @@ require (
github.com/sergi/go-diff v1.2.0 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stackrox/istio-cves v0.0.0-20221007013142-0bde9b541ec8
github.com/tkuchiki/go-timezone v0.2.2 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/transparency-dev/merkle v0.0.1 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1003,6 +1003,8 @@ github.com/stackrox/docker-registry-client v0.0.0-20220204234128-07f109db0819 h1
github.com/stackrox/docker-registry-client v0.0.0-20220204234128-07f109db0819/go.mod h1:tEU0CD7y2mq9HAWFZY48THyKPFy6oMv19UT5bnTvrRo=
github.com/stackrox/dotnet-scraper v0.0.0-20201023051640-72ef543323dd h1:vEjp7Q66zd4W72//Uk3uyVN50Mh/nFLbN9pb7CVK7mE=
github.com/stackrox/dotnet-scraper v0.0.0-20201023051640-72ef543323dd/go.mod h1:HILeV3i/EyJz844GcrC3+oU7oZONhjfujaIYBMJ/bZE=
github.com/stackrox/istio-cves v0.0.0-20221007013142-0bde9b541ec8 h1:rUIvoAHokPcd92aJT2gJwVeyE8tMuaqS5l5s3cEgXFY=
github.com/stackrox/istio-cves v0.0.0-20221007013142-0bde9b541ec8/go.mod h1:ZF7mH4kH1G+82HxR3uFDHvyLG8eCOdrh1RDyQcTGkBA=
github.com/stackrox/k8s-cves v0.0.0-20220818200547-7d0d1420c58d h1:88Iui7fSMKgXvpyfBlbP3qosrqyv3qMgOJ6JJ4V4tFQ=
github.com/stackrox/k8s-cves v0.0.0-20220818200547-7d0d1420c58d/go.mod h1:GJwFpFwCxiYhgpJWrAkM+v9Z9gpgtyWxkRdK4JjsOIg=
github.com/stackrox/nvdtools v0.0.0-20220608171543-e758756071a0 h1:hLexaI/zJBDP4OlxN1za3IJM7cfH+Kie7F/wdWn3xGA=
Expand Down
Loading