Skip to content

Commit 111f6f5

Browse files
dcaraveldcaravel
authored
ROX-26604: Add layer lineage to RHEL v2 data model (#1720)
1 parent 2124e53 commit 111f6f5

12 files changed

+231
-47
lines changed

‎api/v1/models.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ func LayerFromDatabaseModel(db database.Datastore, dbLayer database.Layer, linea
143143
layer.Features = append(layer.Features, *feature)
144144
}
145145
if !uncertifiedRHEL && namespaces.IsRHELNamespace(layer.NamespaceName) {
146-
certified, err := addRHELv2Vulns(db, &layer)
146+
certified, err := addRHELv2Vulns(db, &layer, lineage)
147147
if err != nil {
148148
return layer, notes, err
149149
}
@@ -212,7 +212,7 @@ func ComponentsFromDatabaseModel(db database.Datastore, dbLayer *database.Layer,
212212
if !uncertifiedRHEL && namespaces.IsRHELNamespace(namespaceName) {
213213
var certified bool
214214
var err error
215-
rhelv2PkgEnvs, certified, err = getRHELv2PkgEnvs(db, dbLayer.Name)
215+
rhelv2PkgEnvs, certified, err = getRHELv2PkgEnvs(db, dbLayer.Name, lineage)
216216
if err != nil {
217217
return nil, err
218218
}

‎api/v1/models_rhelv2.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@ const (
2929
// certified as part of Red Hat's Scanner Certification Program.
3030
// The returned bool indicates if full certified scanning was performed.
3131
// This is typically only `false` for images without proper CPE information.
32-
func addRHELv2Vulns(db database.Datastore, layer *Layer) (bool, error) {
33-
pkgEnvs, cpesExist, err := getRHELv2PkgEnvs(db, layer.Name)
32+
func addRHELv2Vulns(db database.Datastore, layer *Layer, lineage string) (bool, error) {
33+
pkgEnvs, cpesExist, err := getRHELv2PkgEnvs(db, layer.Name, lineage)
3434
if err != nil {
3535
return false, err
3636
}
@@ -197,8 +197,8 @@ func shareCPEs(layers []*database.RHELv2Layer) bool {
197197
}
198198

199199
// getRHELv2PkgEnvs returns a map from package ID to package environment and a bool to indicate CPEs exist in the image.
200-
func getRHELv2PkgEnvs(db database.Datastore, layerName string) (map[int]*database.RHELv2PackageEnv, bool, error) {
201-
layers, err := db.GetRHELv2Layers(layerName)
200+
func getRHELv2PkgEnvs(db database.Datastore, layerName, layerLineage string) (map[int]*database.RHELv2PackageEnv, bool, error) {
201+
layers, err := db.GetRHELv2Layers(layerName, layerLineage)
202202
if err != nil {
203203
return nil, false, err
204204
}

‎api/v1/models_rhelv2_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func newMockRHELv2Datastore() *mockRHELv2Datastore {
2929
layers: make(map[string][]*database.RHELv2Layer),
3030
vulns: make(map[int][]*database.RHELv2Vulnerability),
3131
}
32-
db.FctGetRHELv2Layers = func(layer string) ([]*database.RHELv2Layer, error) {
32+
db.FctGetRHELv2Layers = func(layer, lineage string) ([]*database.RHELv2Layer, error) {
3333
return db.layers[layer], nil
3434
}
3535
db.FctGetRHELv2Vulnerabilities = func(records []*database.RHELv2Record) (map[int][]*database.RHELv2Vulnerability, error) {

‎database/database.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,7 +145,7 @@ type Datastore interface {
145145
// GetRHELv2Layers retrieves the corresponding layers for the image
146146
// represented by the given layer.
147147
// The returned slice is sorted in order from base layer to top.
148-
GetRHELv2Layers(layer string) ([]*RHELv2Layer, error)
148+
GetRHELv2Layers(layer, lineage string) ([]*RHELv2Layer, error)
149149

150150
// GetRHELv2Vulnerabilities retrieves RHELv2 vulnerabilities based on the given records.
151151
// The returned value maps package ID to the related vulnerabilities.

‎database/mock.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ type MockDatastore struct {
3131
FctFindLayer func(name, lineage string, opts *DatastoreOptions) (Layer, error)
3232
FctDeleteLayer func(name string) error
3333
FctInsertRHELv2Layer func(*RHELv2Layer) error
34-
FctGetRHELv2Layers func(layer string) ([]*RHELv2Layer, error)
34+
FctGetRHELv2Layers func(layer, lineage string) ([]*RHELv2Layer, error)
3535
FctGetRHELv2Vulnerabilities func(records []*RHELv2Record) (map[int][]*RHELv2Vulnerability, error)
3636
FctListVulnerabilities func(namespaceName string, limit int, page int) ([]Vulnerability, int, error)
3737
FctInsertVulnerabilities func(vulnerabilities []Vulnerability) error
@@ -82,9 +82,9 @@ func (mds *MockDatastore) InsertRHELv2Layer(layer *RHELv2Layer) error {
8282
panic("required mock function not implemented")
8383
}
8484

85-
func (mds *MockDatastore) GetRHELv2Layers(layer string) ([]*RHELv2Layer, error) {
85+
func (mds *MockDatastore) GetRHELv2Layers(layer, layerLineage string) ([]*RHELv2Layer, error) {
8686
if mds.FctGetRHELv2Layers != nil {
87-
return mds.FctGetRHELv2Layers(layer)
87+
return mds.FctGetRHELv2Layers(layer, layerLineage)
8888
}
8989
panic("required mock function not implemented")
9090
}

‎database/models.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -205,11 +205,13 @@ func (p *RHELv2Package) GetPackageVersion() string {
205205
type RHELv2Layer struct {
206206
Model
207207

208-
Hash string
209-
ParentHash string
210-
Dist string
211-
Pkgs []*RHELv2Package
212-
CPEs []string
208+
Hash string
209+
ParentHash string
210+
Dist string
211+
Pkgs []*RHELv2Package
212+
CPEs []string
213+
Lineage string
214+
ParentLineage string
213215
}
214216

215217
// RHELv2Components defines the RHELv2 components found in a layer.

‎database/pgsql/migrations/00021_rhel_layer_lineage.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package migrations
2+
3+
import "github.com/remind101/migrate"
4+
5+
func init() {
6+
RegisterMigration(migrate.Migration{
7+
ID: 21,
8+
Up: migrate.Queries([]string{
9+
10+
// The lineage column mimics the existing `layer` table. The parent_lineage column is used
11+
// instead of equivalent parent_id column from the 'layer' table to avoid an extra query
12+
// on insert (which would be necessary to determine the parent id).
13+
`ALTER TABLE rhelv2_layer ADD COLUMN IF NOT EXISTS lineage varchar;`,
14+
`ALTER TABLE rhelv2_layer ADD COLUMN IF NOT EXISTS parent_lineage varchar`,
15+
16+
// Create a new unique constraint that includes lineage (and drop the old constraint)
17+
`ALTER TABLE rhelv2_layer ADD CONSTRAINT rhelv2_layer_hash_lineage_key UNIQUE (hash, lineage)`,
18+
`ALTER TABLE rhelv2_layer DROP CONSTRAINT IF EXISTS rhelv2_layer_hash_key`,
19+
20+
// Create additional index to improve performance when recursively traversing parents.
21+
`CREATE INDEX IF NOT EXISTS rhelv2_layer_parent_idx on rhelv2_layer (parent_hash, parent_lineage)`,
22+
}),
23+
})
24+
}

‎database/pgsql/queries.go

Lines changed: 14 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -264,9 +264,9 @@ const (
264264
deleteStaleRHELv2Vulns = `DELETE FROM vuln_v2 WHERE name = ANY($1::text[]) and package_name = ANY($2::text[]) and cpe = ANY($3::text[]) and package_module = $4;`
265265

266266
insertRHELv2Layer = `
267-
INSERT INTO rhelv2_layer (hash, parent_hash, dist, cpes)
268-
VALUES ($1, $2, $3, $4)
269-
ON CONFLICT (hash) DO NOTHING;`
267+
INSERT INTO rhelv2_layer (hash, parent_hash, dist, cpes, lineage, parent_lineage)
268+
VALUES ($1, $2, $3, $4, $5, $6)
269+
ON CONFLICT (hash, lineage) DO NOTHING;`
270270

271271
// Inside the `WITH RECURSIVE`, the base case is the top query, and the
272272
// recursive case is the bottom query.
@@ -276,15 +276,17 @@ const (
276276
// This query looks for all the layers in the given layer's hierarchy.
277277
searchRHELv2Layers = `
278278
WITH RECURSIVE layers AS (
279-
SELECT id, hash, parent_hash, dist, cpes
279+
SELECT id, hash, parent_hash, dist, cpes, lineage, parent_lineage
280280
FROM rhelv2_layer
281-
WHERE hash = $1
282-
UNION
283-
SELECT l.id, l.hash, l.parent_hash, l.dist, l.cpes
281+
WHERE hash = $1
282+
AND lineage = $2
283+
UNION
284+
SELECT l.id, l.hash, l.parent_hash, l.dist, l.cpes, l.lineage, l.parent_lineage
284285
FROM layers ll, rhelv2_layer l
285286
WHERE ll.parent_hash = l.hash
287+
AND ll.parent_lineage = l.lineage
286288
)
287-
SELECT id, hash, dist, cpes
289+
SELECT id, hash, dist, cpes, lineage
288290
FROM layers;`
289291

290292
insertRHELv2Package = `
@@ -304,7 +306,8 @@ const (
304306
layer AS (
305307
SELECT id AS layer_id
306308
FROM rhelv2_layer
307-
WHERE rhelv2_layer.hash = $5
309+
WHERE rhelv2_layer.hash = $5
310+
AND rhelv2_layer.lineage = $6
308311
)
309312
INSERT
310313
INTO rhelv2_package_scanartifact (layer_id, package_id)
@@ -324,7 +327,8 @@ const (
324327
rhelv2_package_scanartifact
325328
LEFT JOIN rhelv2_package ON
326329
rhelv2_package_scanartifact.package_id = rhelv2_package.id
327-
JOIN rhelv2_layer ON rhelv2_layer.hash = $1
330+
JOIN rhelv2_layer ON rhelv2_layer.hash = $1
331+
AND rhelv2_layer.lineage = $2
328332
WHERE
329333
rhelv2_package_scanartifact.layer_id = rhelv2_layer.id;`
330334

‎database/pgsql/rhelv2_layer.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
3030
return err
3131
}
3232

33-
if err := pgSQL.insertRHELv2Packages(tx, layer.Hash, layer.Pkgs); err != nil {
33+
if err := pgSQL.insertRHELv2Packages(tx, layer.Hash, layer.Pkgs, layer.Lineage); err != nil {
3434
utils.IgnoreError(tx.Rollback)
3535
return err
3636
}
@@ -46,11 +46,11 @@ func (pgSQL *pgSQL) InsertRHELv2Layer(layer *database.RHELv2Layer) error {
4646
func (pgSQL *pgSQL) insertRHELv2Layer(tx *sql.Tx, layer *database.RHELv2Layer) error {
4747
defer metrics.ObserveQueryTime("insertRHELv2Layer", "layer", time.Now())
4848

49-
_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs))
49+
_, err := tx.Exec(insertRHELv2Layer, layer.Hash, layer.ParentHash, layer.Dist, pq.Array(layer.CPEs), layer.Lineage, layer.ParentLineage)
5050
return err
5151
}
5252

53-
func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package) error {
53+
func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*database.RHELv2Package, lineage string) error {
5454
// Sort packages to avoid potential deadlock.
5555
// Sort by the unique index (name, version, module, arch).
5656
sort.SliceStable(pkgs, func(i, j int) bool {
@@ -91,6 +91,7 @@ func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*datab
9191
pkg.Module,
9292
pkg.Arch,
9393
layer,
94+
lineage,
9495
)
9596

9697
if err != nil {
@@ -101,7 +102,7 @@ func (pgSQL *pgSQL) insertRHELv2Packages(tx *sql.Tx, layer string, pkgs []*datab
101102
return nil
102103
}
103104

104-
func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer, error) {
105+
func (pgSQL *pgSQL) GetRHELv2Layers(layerHash, layerLineage string) ([]*database.RHELv2Layer, error) {
105106
defer metrics.ObserveQueryTime("getRHELv2Layers", "all", time.Now())
106107

107108
tx, err := pgSQL.BeginTx(context.Background(), &sql.TxOptions{
@@ -111,7 +112,7 @@ func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer,
111112
return nil, handleError("GetRHELv2Layers.Begin()", err)
112113
}
113114

114-
rows, err := tx.Query(searchRHELv2Layers, layerHash)
115+
rows, err := tx.Query(searchRHELv2Layers, layerHash, layerLineage)
115116
if err != nil {
116117
return nil, err
117118
}
@@ -124,7 +125,7 @@ func (pgSQL *pgSQL) GetRHELv2Layers(layerHash string) ([]*database.RHELv2Layer,
124125
rhelv2Layer database.RHELv2Layer
125126
cpes []string
126127
)
127-
if err := rows.Scan(&rhelv2Layer.ID, &rhelv2Layer.Hash, &rhelv2Layer.Dist, pq.Array(&cpes)); err != nil {
128+
if err := rows.Scan(&rhelv2Layer.ID, &rhelv2Layer.Hash, &rhelv2Layer.Dist, pq.Array(&cpes), &rhelv2Layer.Lineage); err != nil {
128129
utils.IgnoreError(tx.Rollback)
129130
return nil, err
130131
}
@@ -176,7 +177,7 @@ func (pgSQL *pgSQL) populatePackages(tx *sql.Tx, layers []*database.RHELv2Layer)
176177
func (pgSQL *pgSQL) getPackagesByLayer(tx *sql.Tx, layer *database.RHELv2Layer) error {
177178
defer metrics.ObserveQueryTime("getRHELv2Layers", "packagesByLayer", time.Now())
178179

179-
rows, err := tx.Query(searchRHELv2Package, layer.Hash)
180+
rows, err := tx.Query(searchRHELv2Package, layer.Hash, layer.Lineage)
180181
if err != nil {
181182
return err
182183
}

0 commit comments

Comments
 (0)