ROX-6988: Fix RHEL updates by deleting old CVEs when RH advisories are updated

.circleci/config.yml

@@ -415,6 +415,7 @@ commands:
 
       - run:
           name: Run sanity tests
+          no_output_timeout: 20m
           command: |
             make e2e-tests
 
@@ -674,6 +675,9 @@ jobs:
           paths:
             - diff-dumps
 
+      - store_artifacts:
+          path: /tmp/diff-dumps
+
   upload-diff-dumps-and-offline-dumps:
     <<: *defaults
     steps:

cmd/updater/diffdumps/cmd.go

@@ -308,6 +308,7 @@ type config struct {
 	UseDPKGParserForAlpine     bool `json:"useDPKGParserForAlpine"`
 	// SkipRHELv2TitleComparison need only be set when SkipRHELv2Vulns is NOT set.
 	SkipRHELv2TitleComparison bool `json:"skipRHELv2TitleComparison"`
+	RemovedUnusedRHELv2CPEs   bool `json:"removedUnusedRHELv2CPEs"`
 }
 
 // Command defines the diff-dumps command.

cmd/updater/diffdumps/rhelv2_diff.go

@@ -11,6 +11,7 @@ import (
 	"github.com/mitchellh/hashstructure"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/stackrox/rox/pkg/set"
 	"github.com/stackrox/rox/pkg/utils"
 	"github.com/stackrox/scanner/database"
 	"github.com/stackrox/scanner/pkg/repo2cpe"
@@ -87,6 +88,23 @@ func generateRHELv2Diff(cfg config, outputDir string, baseLastModifiedTime time.
 			headVuln.Title = ""
 		}
 
+		// If we did not remove the unused RHEL CPEs then we just want to check that no new CPEs were added
+		// otherwise, we can do a set match
+		baseCPEs := set.NewStringSet(matchingBaseVuln.CPEs...)
+		headCPEs := set.NewStringSet(headVuln.CPEs...)
+		if !cfg.RemovedUnusedRHELv2CPEs {
+			// There were no CVEs added for previous scanners
+			headVuln.SubCVEs = nil
+
+			if len(headCPEs.Difference(baseCPEs)) > 0 {
+				filtered = append(filtered, headVuln)
+				continue
+			}
+		} else if !headCPEs.Equal(baseCPEs) {
+			filtered = append(filtered, headVuln)
+			continue
+		}
+
 		headHash, err := vulnHash(headVuln)
 		if err != nil {
 			log.Warnf("Unable to hash new vuln %s. Skipping head vuln...", headVuln.Name)
@@ -135,6 +153,7 @@ func generateRHELv2VulnsDiff(cfg config, outputDir string, baseLastModifiedTime
 	// Let's us know if the base dump had RHELv2 data.
 	rhelExists := len(baseFiles) > 0
 
+	// Get RHEL Repo to CPE file
 	repoToCPEFile := filepath.Join(vulndump.RHELv2DirName, repo2cpe.RHELv2CPERepoName)
 	for _, headF := range headZipR.File {
 		name := headF.Name

database/models.go

@@ -139,8 +139,9 @@ type RHELv2Vulnerability struct {
 	Severity     string               `json:"severity"`
 	CVSSv3       string               `json:"cvssv3,omitempty"`
 	CVSSv2       string               `json:"cvssv2,omitempty"`
-	CPEs         []string             `json:"cpes" hash:"set"`
+	CPEs         []string             `json:"cpes" hash:"ignore"` // These are checked explcitly due to the removal of unused CPEs
 	PackageInfos []*RHELv2PackageInfo `json:"package_info" hash:"set"`
+	SubCVEs      []string             `json:"sub_cves,omitempty" hash:"set"`
 }
 
 // RHELv2PackageInfo defines all the data necessary for fully define a RHELv2 package.

database/pgsql/queries.go

@@ -312,6 +312,8 @@ const (
 		WHERE
 			rhelv2_package_scanartifact.layer_id = rhelv2_layer.id;`
 
+	deleteStaleRHELv2CVEs = `DELETE FROM vuln_package WHERE name = ANY($1::text[]) and package_name = ANY($2::text[]) and cpe = ANY($3::text[]) and package_module = $4;`
+
 	///////////////////////////////////////////////////
 	// END
 	// Influenced by ClairCore under Apache 2.0 License

database/pgsql/rhelv2_vulnerability.go

@@ -12,20 +12,22 @@ import (
 	"database/sql"
 	"time"
 
+	"github.com/lib/pq"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/stackrox/rox/pkg/batcher"
 	"github.com/stackrox/rox/pkg/set"
 	"github.com/stackrox/rox/pkg/utils"
 	"github.com/stackrox/scanner/database"
 	"github.com/stackrox/scanner/database/metrics"
+	"github.com/stackrox/scanner/pkg/rhelv2"
 )
 
 const (
 	rhelv2InsertionBatchSize = 50
 )
 
-func (pgSQL *pgSQL) insertRHELv2Vulnerability(vulnStatement, vulnPackageStatement *sql.Stmt, vuln *database.RHELv2Vulnerability) error {
+func (pgSQL *pgSQL) insertRHELv2Vulnerability(vulnStatement, vulnPackageStatement, vulnDeleteStatement *sql.Stmt, vuln *database.RHELv2Vulnerability) error {
 	vulnHash := md5Vuln(vuln)
 	_, err := vulnStatement.Exec(
 		vulnHash,
@@ -36,6 +38,32 @@ func (pgSQL *pgSQL) insertRHELv2Vulnerability(vulnStatement, vulnPackageStatemen
 		return errors.Wrapf(err, "inserting RHELv2 vuln %q into the DB", vuln.Name)
 	}
 
+	if rhelv2.IsRedHatAdvisory(vuln.Name) {
+		if len(vuln.SubCVEs) != 0 && len(vuln.PackageInfos) != 0 && len(vuln.CPEs) != 0 {
+			packagesByModule := make(map[string][]string)
+			for _, pkgInfo := range vuln.PackageInfos {
+				for _, pkg := range pkgInfo.Packages {
+					packagesByModule[pkg.Module] = append(packagesByModule[pkg.Module], pkg.Name)
+				}
+			}
+			for module, packages := range packagesByModule {
+				result, err := vulnDeleteStatement.Exec(pq.Array(vuln.SubCVEs), pq.Array(packages), pq.Array(vuln.CPEs), module)
+				if err != nil {
+					return errors.Wrapf(err, "deleting stale rhel v2 CVEs")
+				}
+				if log.IsLevelEnabled(log.DebugLevel) {
+					affected, err := result.RowsAffected()
+					if err != nil {
+						return errors.Wrapf(err, "getting rows affected")
+					}
+					if affected != 0 {
+						log.Debugf("removed %d vulnerability entries for %s", affected, vuln.Name)
+					}
+				}
+			}
+		}
+	}
+
 	for _, pkgInfo := range vuln.PackageInfos {
 		for _, pkg := range pkgInfo.Packages {
 			for _, cpe := range vuln.CPEs {
@@ -53,11 +81,10 @@ func (pgSQL *pgSQL) insertRHELv2Vulnerability(vulnStatement, vulnPackageStatemen
 			}
 		}
 	}
-
 	return nil
 }
 
-func (pgSQL *pgSQL) insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt *sql.Stmt, vulnerabilities []*database.RHELv2Vulnerability) error {
+func (pgSQL *pgSQL) insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt, vulnDeleteStmt *sql.Stmt, vulnerabilities []*database.RHELv2Vulnerability) error {
 	tx, err := pgSQL.Begin()
 	if err != nil {
 		return errors.Wrap(err, "unable to start transaction")
@@ -69,12 +96,15 @@ func (pgSQL *pgSQL) insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt *sq
 	vulnPkgStmt = tx.Stmt(vulnPkgStmt)
 	defer utils.IgnoreError(vulnPkgStmt.Close)
 
+	vulnDeleteStmt = tx.Stmt(vulnDeleteStmt)
+	defer utils.IgnoreError(vulnDeleteStmt.Close)
+
 	for _, vulnerability := range vulnerabilities {
 		if len(vulnerability.PackageInfos) == 0 {
 			continue
 		}
 
-		err = pgSQL.insertRHELv2Vulnerability(vulnStmt, vulnPkgStmt, vulnerability)
+		err = pgSQL.insertRHELv2Vulnerability(vulnStmt, vulnPkgStmt, vulnDeleteStmt, vulnerability)
 		if err != nil {
 			utils.IgnoreError(tx.Rollback)
 			return err
@@ -83,6 +113,7 @@ func (pgSQL *pgSQL) insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt *sq
 	if err := tx.Commit(); err != nil {
 		return handleError("insertRHELv2Vulnerability.Commit()", err)
 	}
+
 	return nil
 }
 
@@ -100,11 +131,16 @@ func (pgSQL *pgSQL) InsertRHELv2Vulnerabilities(vulnerabilities []*database.RHEL
 		return err
 	}
 	defer utils.IgnoreError(vulnPkgStmt.Close)
+	vulnDeleteStmt, err := pgSQL.Prepare(deleteStaleRHELv2CVEs)
+	if err != nil {
+		return err
+	}
+	defer utils.IgnoreError(vulnDeleteStmt.Close)
 
 	batch := batcher.New(len(vulnerabilities), rhelv2InsertionBatchSize)
 	for start, end, ok := batch.Next(); ok; start, end, ok = batch.Next() {
 		startTime := time.Now()
-		if err := pgSQL.insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt, vulnerabilities[start:end]); err != nil {
+		if err := pgSQL.insertRHELv2VulnerabilitiesBatched(vulnStmt, vulnPkgStmt, vulnDeleteStmt, vulnerabilities[start:end]); err != nil {
 			return err
 		}
 		metrics.ObserveQueryTime("insertRHELv2Vulnerabilities", "batchInsert", startTime)

database/pgsql/rhelv2_vulnerability_test.go

@@ -8,14 +8,12 @@ import (
 	archop "github.com/quay/claircore"
 	"github.com/stackrox/scanner/database"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestRHELv2Vulnerability(t *testing.T) {
 	datastore, err := openDatabaseForTest("InsertRHELv2Vulnerability", false)
-	if err != nil {
-		t.Error(err)
-		return
-	}
+	require.NoError(t, err)
 	defer datastore.Close()
 
 	v1 := &database.RHELv2Vulnerability{
@@ -189,3 +187,126 @@ func TestRHELv2Vulnerability(t *testing.T) {
 	assert.Len(t, vulns[0], 2)
 	assert.Len(t, vulns[1], 1)
 }
+
+func TestRHELv2VulnerabilityCVERemoval(t *testing.T) {
+	datastore, err := openDatabaseForTest("TestRHELv2VulnerabilityCVERemoval", false)
+	require.NoError(t, err)
+	defer datastore.Close()
+
+	v1 := &database.RHELv2Vulnerability{
+		Name:        "CVE-2021-1234",
+		Description: "my vuln",
+		Link:        "https://access.redhat.com/security/cve/CVE-2021-1234",
+		Severity:    "Important",
+		CVSSv2:      "4.0/AV:N/AC:H/Au:N/C:P/I:P/A:N",
+		CPEs: []string{
+			"cpe:/o:redhat:enterprise_linux:8",
+		},
+		PackageInfos: []*database.RHELv2PackageInfo{
+			{
+				FixedInVersion: "0:1.0.8-13.el8",
+				ArchOperation:  archop.OpEquals,
+				Packages: []*database.RHELv2Package{
+					{
+						Name: "package",
+						Arch: "x86_64",
+					},
+				},
+			},
+		},
+	}
+	v2 := &database.RHELv2Vulnerability{
+		Name:        "CVE-2021-1235",
+		Title:       "RHSA-2013:0565: Red Hat Enterprise MRG Grid 2.3 security update (Low)",
+		Description: "my vuln2",
+		Link:        "https://access.redhat.com/security/cve/CVE-2021-1235",
+		Severity:    "Moderate",
+		CVSSv3:      "4.6/CVSS:3.0/AV:A/AC:L/PR:L/UI:R/S:U/C:N/I:L/A:L",
+		CPEs: []string{
+			"cpe:/o:redhat:enterprise_linux:8",
+			"cpe:/o:redhat:enterprise_linux:7",
+		},
+		PackageInfos: []*database.RHELv2PackageInfo{
+			{
+				FixedInVersion: "0:1.0.8-13.el8",
+				ArchOperation:  archop.OpEquals,
+				Packages: []*database.RHELv2Package{
+					{
+						Name: "package",
+						Arch: "x86_64",
+					},
+				},
+			},
+		},
+	}
+
+	err = datastore.InsertRHELv2Vulnerabilities([]*database.RHELv2Vulnerability{v1, v2})
+	assert.NoError(t, err)
+
+	// Insert RHSA for CPE cpe:/o:redhat:enterprise_linux:7 with a fix and sub CVE for the two above
+	// Only the second one should be removed as the CPE will match
+	v3 := &database.RHELv2Vulnerability{
+		Name:        "RHSA-2013:0565",
+		Title:       "RHSA-2013:0565: Red Hat Enterprise MRG Grid 2.3 security update (Low)",
+		Description: "my vuln2",
+		Link:        "https://access.redhat.com/security/cve/CVE-2021-1235",
+		Severity:    "Moderate",
+		CVSSv3:      "4.6/CVSS:3.0/AV:A/AC:L/PR:L/UI:R/S:U/C:N/I:L/A:L",
+		CPEs: []string{
+			"cpe:/o:redhat:enterprise_linux:7",
+		},
+		PackageInfos: []*database.RHELv2PackageInfo{
+			{
+				FixedInVersion: "0:1.0.8-13.el8",
+				ArchOperation:  archop.OpEquals,
+				Packages: []*database.RHELv2Package{
+					{
+						Name: "package",
+						Arch: "x86_64",
+					},
+				},
+			},
+		},
+		SubCVEs: []string{"CVE-2021-1234", "CVE-2021-1235"},
+	}
+
+	err = datastore.InsertRHELv2Vulnerabilities([]*database.RHELv2Vulnerability{v3})
+	assert.NoError(t, err)
+
+	records := []*database.RHELv2Record{
+		{
+			Pkg: &database.RHELv2Package{
+				Name: "package",
+			},
+			CPE: "cpe:/o:redhat:enterprise_linux:7",
+		},
+	}
+
+	// Sanity check all fields are valid for single vuln.
+	vulns, err := datastore.GetRHELv2Vulnerabilities(records)
+	assert.NoError(t, err)
+	assert.Len(t, vulns, 1)
+	assert.Len(t, vulns[0], 1)
+	vuln := vulns[0][0]
+	assert.Equal(t, v3.Name, vuln.Name)
+
+	records = []*database.RHELv2Record{
+		{
+			Pkg: &database.RHELv2Package{
+				Model: database.Model{ID: 0},
+				Name:  "package",
+			},
+			CPE: "cpe:/o:redhat:enterprise_linux:8",
+		},
+	}
+
+	// Ensure both vulns are returned.
+	vulns, err = datastore.GetRHELv2Vulnerabilities(records)
+	assert.NoError(t, err)
+	assert.Len(t, vulns, 1)
+	vulnNames := make([]string, 0, len(vulns[0]))
+	for _, vuln := range vulns[0] {
+		vulnNames = append(vulnNames, vuln.Name)
+	}
+	assert.ElementsMatch(t, vulnNames, []string{v1.Name, v2.Name})
+}

pkg/cpeutils/utils.go

@@ -27,3 +27,8 @@ func compareAttributes(c1, c2 wfn.AttributesWithFixedIn) int {
 	}
 	return strings.Compare(c1.Version, c2.Version)
 }
+
+// IsOpenShiftCPE returns if the passed CPE is a known OpenShift CPE
+func IsOpenShiftCPE(cpe string) bool {
+	return strings.HasPrefix(cpe, "cpe:/a:redhat:openshift:")
+}