Louis/layer scanned fix

Makefile

@@ -1,6 +1,11 @@
 docker ?= docker
 docker-compose ?= docker-compose
 
+# clears any go code in various caches
+.PHONY: clear-cache
+clear-cache:
+	go clean -cache -testcache -modcache
+
 # generates mocks of interfaces for testing
 .PHONY: genmocks
 genmocks:

alpine/distributionscanner.go

@@ -90,7 +90,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated alpine release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -111,7 +111,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all alpine release regexp and returns the associated

aws/distributionscanner.go

@@ -60,7 +60,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated AWS release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -83,7 +83,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all AWS release regexp and returns the associated

debian/distributionscanner.go

@@ -65,7 +65,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated Debian release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -86,7 +86,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all Debian release regexp and returns the associated

internal/indexer/layerscanner/layerscanner.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/internal/indexer"
+	"github.com/rs/zerolog"
 )
 
 // layerScanner implements the indexer.LayerScanner interface.
@@ -116,6 +117,13 @@ func (ls *layerScanner) Scan(ctx context.Context, manifest claircore.Digest, lay
 }
 
 func (ls *layerScanner) scanPackages(ctx context.Context, layer *claircore.Layer, s indexer.PackageScanner) error {
+	log := zerolog.Ctx(ctx).With().
+		Str("component", "internal/indexer/layerscannner/layerScanner.scanPackages").
+		Str("scanner", s.Name()).
+		Str("layer", layer.Hash.String()).
+		Logger()
+
+	log.Debug().Msg("starting package scan")
 	if err := ls.addToken(ctx); err != nil {
 		return err
 	}
@@ -126,26 +134,43 @@ func (ls *layerScanner) scanPackages(ctx context.Context, layer *claircore.Layer
 		return err
 	}
 	if ok {
+		log.Debug().Msg("layer already scanned")
 		return nil
 	}
 
 	v, err := s.Scan(ctx, layer)
 	if err != nil {
 		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
 	}
+	err = ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	if err != nil {
+		return fmt.Errorf("could not set layer scanned: %v", layer)
+	}
+
 	if v == nil {
+		log.Debug().Msg("scan returned a nil")
 		return nil
 	}
 
-	err = ls.Store.IndexPackages(ctx, v, layer, s)
-	if err != nil {
-		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+	if len(v) > 0 {
+		log.Debug().Int("count", len(v)).Msg("scan returned packages")
+		err = ls.Store.IndexPackages(ctx, v, layer, s)
+		if err != nil {
+			return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+		}
 	}
 
-	return ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	return nil
 }
 
 func (ls *layerScanner) scanDists(ctx context.Context, layer *claircore.Layer, s indexer.DistributionScanner) error {
+	log := zerolog.Ctx(ctx).With().
+		Str("component", "internal/indexer/layerscannner/layerScanner.scanDists").
+		Str("scanner", s.Name()).
+		Str("layer", layer.Hash.String()).
+		Logger()
+
+	log.Debug().Msg("starting dist scan")
 	if err := ls.addToken(ctx); err != nil {
 		return err
 	}
@@ -156,26 +181,43 @@ func (ls *layerScanner) scanDists(ctx context.Context, layer *claircore.Layer, s
 		return err
 	}
 	if ok {
+		log.Debug().Msg("layer already scanned")
 		return nil
 	}
 
 	v, err := s.Scan(ctx, layer)
 	if err != nil {
 		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
 	}
+	err = ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	if err != nil {
+		return fmt.Errorf("could not set layer scanned: %+v %+v", layer, s)
+	}
+
 	if v == nil {
+		log.Debug().Msg("scan returned a nil")
 		return nil
 	}
 
-	err = ls.Store.IndexDistributions(ctx, v, layer, s)
-	if err != nil {
-		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+	if len(v) > 0 {
+		log.Debug().Int("count", len(v)).Msg("scan returned dists")
+		err = ls.Store.IndexDistributions(ctx, v, layer, s)
+		if err != nil {
+			return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+		}
 	}
 
-	return ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	return nil
 }
 
 func (ls *layerScanner) scanRepos(ctx context.Context, layer *claircore.Layer, s indexer.RepositoryScanner) error {
+	log := zerolog.Ctx(ctx).With().
+		Str("component", "internal/indexer/layerscannner/layerScanner.scanRepos").
+		Str("scanner", s.Name()).
+		Str("layer", layer.Hash.String()).
+		Logger()
+
+	log.Debug().Msg("starting repo scan")
 	if err := ls.addToken(ctx); err != nil {
 		return err
 	}
@@ -186,21 +228,31 @@ func (ls *layerScanner) scanRepos(ctx context.Context, layer *claircore.Layer, s
 		return err
 	}
 	if ok {
+		log.Debug().Msg("layer already scanned")
 		return nil
 	}
 
 	v, err := s.Scan(ctx, layer)
 	if err != nil {
 		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
 	}
+	err = ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	if err != nil {
+		return fmt.Errorf("could not set layer scanned: %v", layer)
+	}
+
 	if v == nil {
+		log.Debug().Msg("scan returned a nil")
 		return nil
 	}
 
-	err = ls.Store.IndexRepositories(ctx, v, layer, s)
-	if err != nil {
-		return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+	if len(v) > 0 {
+		log.Debug().Int("count", len(v)).Msg("scan returned repos")
+		err = ls.Store.IndexRepositories(ctx, v, layer, s)
+		if err != nil {
+			return fmt.Errorf("scanner: %v error: %v", s.Name(), err)
+		}
 	}
 
-	return ls.Store.SetLayerScanned(ctx, layer.Hash, s)
+	return nil
 }

internal/indexer/postgres/layerscanned.go

@@ -2,47 +2,51 @@ package postgres
 
 import (
 	"context"
-	"database/sql"
 	"fmt"
 
+	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/internal/indexer"
 )
 
 func layerScanned(ctx context.Context, pool *pgxpool.Pool, hash claircore.Digest, scnr indexer.VersionedScanner) (bool, error) {
 	const (
-		// this query will return ErrNoRows  if the scanner is not present in the database
-		// (scanner_id: X, layer_hash: null) if the layer was not scanned by the provided scanner
-		// (scanner_id: X, layer_hash: XYZ)  if the layer has been scanned by the provided scnr
+		selectScanner = `
+			SELECT id
+			FROM scanner
+			WHERE name = $1
+			  AND version = $2
+			  AND kind = $3;
+			`
 		selectScanned = `
-		SELECT id      AS scanner_id,
-			   CASE
-				   WHEN (id IS NOT null) THEN
-					   (SELECT layer_hash FROM scanned_layer WHERE layer_hash = $1 AND scanner_id = scanner_id)
-				   END AS layer_hash
-		FROM scanner
-		WHERE name = $2
-		  AND version = $3
-		  AND kind = $4;
-		`
+			SELECT layer_hash
+			FROM scanned_layer
+			WHERE layer_hash = $1
+			  AND scanner_id = $2
+			`
 	)
 
 	var scannerID int64
-	var layerHash claircore.Digest
-
-	row := pool.QueryRow(ctx, selectScanned, hash, scnr.Name(), scnr.Version(), scnr.Kind())
-	err := row.Scan(&scannerID, &layerHash)
+	row := pool.QueryRow(ctx, selectScanner, scnr.Name(), scnr.Version(), scnr.Kind())
+	err := row.Scan(&scannerID)
 	if err != nil {
-		if err == sql.ErrNoRows {
-			// TODO: make error type to handle this case
-			return false, fmt.Errorf("scanner not found in store: %v", scnr)
+		if err == pgx.ErrNoRows {
+			return false, fmt.Errorf("scanner name and version not found in store: %+v", scnr)
 		}
 		return false, err
 	}
 
-	if layerHash.String() == "" {
-		return false, nil
+	var layerHash string
+	row = pool.QueryRow(ctx, selectScanned, hash.String(), scannerID)
+	err = row.Scan(&layerHash)
+	if err != nil {
+		if err == pgx.ErrNoRows {
+			return false, nil
+		}
+		return false, err
 	}
+
 	return true, nil
+
 }

oracle/distributionscanner.go

@@ -74,7 +74,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated Oracle release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -95,7 +95,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all Oracle release regexp and returns the associated

photon/distributionscanner.go

@@ -67,7 +67,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated photon release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -88,7 +88,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return []*claircore.Distribution{&claircore.Distribution{}}, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all photon release regexp and returns the associated

rhel/distributionscanner.go

@@ -78,7 +78,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated Oracle release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -99,7 +99,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all Oracle release regexp and returns the associated

suse/distributionscanner.go

@@ -80,7 +80,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated Suse release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -101,7 +101,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all Suse release regexp and returns the associated

ubuntu/distributionscanner.go

@@ -85,7 +85,7 @@ func (*DistributionScanner) Kind() string { return scannerKind }
 // and perform a regex match for keywords indicating the associated Ubuntu release
 //
 // If neither file is found a (nil,nil) is returned.
-// If the files are found but all regexp fail to match an empty distribution is returned.
+// If the files are found but all regexp fail to match an empty slice is returned.
 func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Distribution, error) {
 	defer trace.StartRegion(ctx, "Scanner.Scan").End()
 	log := zerolog.Ctx(ctx).With().
@@ -106,7 +106,7 @@ func (ds *DistributionScanner) Scan(ctx context.Context, l *claircore.Layer) ([]
 			return []*claircore.Distribution{dist}, nil
 		}
 	}
-	return nil, nil
+	return []*claircore.Distribution{}, nil
 }
 
 // parse attempts to match all Ubuntu release regexp and returns the associated