dev: todo: indexer data API

datastore/postgres/additionaldata.go

@@ -0,0 +1,64 @@
+package postgres
+
+import (
+	"context"
+	"fmt"
+	"io"
+
+	"github.com/jackc/pgx/v4"
+	"github.com/quay/claircore/updater/driver/v1"
+)
+
+func (s *IndexerStore) GetData(ctx context.Context, namespace string, key string, dec func(context.Context, io.Reader) (any, error)) (any, error) {
+	return dec(ctx, nil) // I don't remember how this works
+}
+
+// TODO(crozzy): accept an iter here?
+// TODO(crozzy): add tracing
+func (s *IndexerStore) UpdateIndexerData(ctx context.Context, _ driver.Fingerprint, ds []driver.IndexerData) error {
+	const (
+		upsertAdditionalData = `
+		INSERT INTO additional_data (
+			namespace,
+			lookup_key,
+			data
+		) VALUES (
+			$1,
+			$2,
+			$3 
+		)
+		ON CONFLICT (namespace, lookup_key)
+		DO UPDATE SET data = EXCLUDED.data;
+		`
+	)
+	tx, err := s.pool.Begin(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create transaction: %w", err)
+	}
+	defer tx.Rollback(ctx)
+
+	for _, d := range ds {
+		// Deal with large object
+		// TODO(crozzy): How to check for duplicate large objects in this process?
+		lo := tx.LargeObjects()
+		oid, err := lo.Create(ctx, 0)
+		if err != nil {
+			return fmt.Errorf("failed to create large object: %w", err)
+		}
+		obj, err := lo.Open(ctx, oid, pgx.LargeObjectModeWrite)
+		if err != nil {
+			return fmt.Errorf("failed to open large object: %w", err)
+		}
+		if _, err = obj.Write(d.Value); err != nil {
+			return fmt.Errorf("failed to write large object: %w", err)
+		}
+		if _, err := tx.Exec(ctx, upsertAdditionalData, d.Namespace, d.Key, oid); err != nil {
+			return fmt.Errorf("failed to upsert additional data: %w", err)
+		}
+	}
+	if err = tx.Commit(ctx); err != nil {
+		return fmt.Errorf("failed to commit transaction: %w", err)
+	}
+
+	return nil
+}

datastore/postgres/migrations/indexer/09-indexer-data.sql

@@ -0,0 +1,24 @@
+-- additional_data
+CREATE EXTENSION IF NOT EXISTS lo;
+CREATE TABLE IF NOT EXISTS additional_data (
+    namespace TEXT,
+    lookup_key TEXT,
+    data lo
+);
+CREATE UNIQUE INDEX IF NOT EXISTS additional_data_unique_idx ON additional_data (namespace, lookup_key);
+
+-- Trigger needed to clean up orphaned objects
+-- see: https://www.postgresql.org/docs/current/lo.html#LO-HOW-TO-USE
+-- Can't IF NOT EXISTS a trigger, so have to manually delete and recreate.
+DO $$
+BEGIN
+    -- Drop the trigger if it exists
+    IF EXISTS (SELECT 1 FROM pg_trigger WHERE tgname = 't_data') THEN
+        DROP TRIGGER t_data ON additional_data;
+    END IF;
+
+    -- Create the trigger
+    CREATE TRIGGER t_data BEFORE UPDATE OR DELETE ON additional_data
+    FOR EACH ROW EXECUTE FUNCTION lo_manage(data);
+END;
+$$ LANGUAGE plpgsql;

datastore/postgres/migrations/migrations.go

@@ -61,6 +61,10 @@ var IndexerMigrations = []migrate.Migration{
 		ID: 8,
 		Up: runFile("indexer/08-index-manifest_layer.sql"),
 	},
+	{
+		ID: 9,
+		Up: runFile("indexer/09-indexer-data.sql"),
+	},
 }
 
 var MatcherMigrations = []migrate.Migration{

indexer/layerscanner.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"iter"
 	"net"
 	"runtime"
 
@@ -201,7 +202,7 @@ func (ls *LayerScanner) scanLayer(ctx context.Context, l *claircore.Layer, s Ver
 	}
 
 	var result result
-	if err := result.Do(ctx, s, l); err != nil {
+	if err := result.Do(ctx, s, l, ls.store); err != nil {
 		return err
 	}
 
@@ -227,7 +228,7 @@ type result struct {
 // Do asserts the Scanner back to having a Scan method, and then calls it.
 //
 // The success value is captured and the error value is returned by Do.
-func (r *result) Do(ctx context.Context, s VersionedScanner, l *claircore.Layer) error {
+func (r *result) Do(ctx context.Context, s VersionedScanner, l *claircore.Layer, dl DataLookup) error {
 	var err error
 	switch s := s.(type) {
 	case PackageScanner:
@@ -243,6 +244,25 @@ func (r *result) Do(ctx context.Context, s VersionedScanner, l *claircore.Layer)
 		r.repos, err = s.Scan(ctx, l)
 	case FileScanner:
 		r.files, err = s.Scan(ctx, l)
+	case ComplexPackageDetector:
+		// TODO (crozzy): separate generic function here?
+		var ps iter.Seq2[claircore.Package, error]
+		ps, err = s.Analyze(ctx, dl, l)
+		for p, iErr := range ps {
+			if iErr != nil {
+				zlog.Warn(ctx).Str("detector", s.Name()).Err(iErr).Msg("error detecting package")
+			}
+			r.pkgs = append(r.pkgs, &p)
+		}
+	case ComplexRepositoryDetector:
+		var repos iter.Seq2[claircore.Repository, error]
+		repos, err = s.Analyze(ctx, dl, l)
+		for repo, iErr := range repos {
+			if iErr != nil {
+				zlog.Warn(ctx).Str("detector", s.Name()).Err(iErr).Msg("error detecting repository")
+			}
+			r.repos = append(r.repos, &repo)
+		}
 	default:
 		panic(fmt.Sprintf("programmer error: unknown type %T used as scanner", s))
 	}

indexer/packagescanner.go

@@ -2,6 +2,7 @@ package indexer
 
 import (
 	"context"
+	"iter"
 
 	"github.com/quay/claircore"
 )
@@ -53,3 +54,8 @@ func NewPackageScannerMock(name, version, kind string) PackageScanner {
 type DefaultRepoScanner interface {
 	DefaultRepository(context.Context) *claircore.Repository
 }
+
+type ComplexPackageDetector interface {
+	VersionedScanner
+	Analyze(context.Context, DataLookup, *claircore.Layer) (iter.Seq2[claircore.Package, error], error)
+}

indexer/repositoryscanner.go

@@ -2,6 +2,7 @@ package indexer
 
 import (
 	"context"
+	"iter"
 
 	"github.com/quay/claircore"
 )
@@ -10,3 +11,8 @@ type RepositoryScanner interface {
 	VersionedScanner
 	Scan(context.Context, *claircore.Layer) ([]*claircore.Repository, error)
 }
+
+type ComplexRepositoryDetector interface {
+	VersionedScanner
+	Analyze(context.Context, DataLookup, *claircore.Layer) (iter.Seq2[claircore.Repository, error], error)
+}

indexer/store.go

@@ -2,8 +2,11 @@ package indexer
 
 import (
 	"context"
+	"io"
 
 	"github.com/quay/claircore"
+
+	"github.com/quay/claircore/updater/driver/v1"
 )
 
 // Store is an interface for dealing with objects libindex needs to persist.
@@ -12,6 +15,7 @@ type Store interface {
 	Setter
 	Querier
 	Indexer
+	DataLookup
 	// Close frees any resources associated with the Store.
 	Close(context.Context) error
 }
@@ -86,3 +90,8 @@ type Indexer interface {
 	// IndexManifest should index the coalesced manifest's content given an IndexReport.
 	IndexManifest(ctx context.Context, ir *claircore.IndexReport) error
 }
+
+type DataLookup interface {
+	GetData(ctx context.Context, namespace string, key string, dec func(context.Context, io.Reader) (any, error)) (any, error)
+	UpdateIndexerData(ctx context.Context, fp driver.Fingerprint, d []driver.IndexerData) error
+}

libindex/libindex.go

@@ -27,6 +27,8 @@
 	"github.com/quay/claircore/rhel/rhcc"
 	"github.com/quay/claircore/rpm"
 	"github.com/quay/claircore/ruby"
+	"github.com/quay/claircore/updater"
+	"github.com/quay/claircore/updater/driver/v1"
 	"github.com/quay/claircore/whiteout"
 )
 
@@ -160,6 +162,38 @@
 		return nil, err
 	}
 
+	if opts.UpdaterConfigs == nil {
+		opts.UpdaterConfigs = make(map[string]driver.ConfigUnmarshaler)
+	}
+
+	u, err := updater.New(ctx, &updater.Options{
+		IndexerStore: l.store,
+		Locker:       l.locker,
+		Client:       l.client,
+		Factories: []driver.UpdaterFactory{
+			&rhel.UpdaterFactory{},
+		},
+		Configs: opts.UpdaterConfigs,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if opts.UpdateInterval < time.Duration(24*time.Hour) {
+		// Update interval is either unset or unrealistically small
+		opts.UpdateInterval = DefaultInterval
+	}
+
+	m := Manager{
+		// TODO(crozzy): Review naming here
+		updater:            u,
+		interval:           opts.UpdateInterval,
+		additionalDataURL:  opts.UpdateBundleURL,
+		additionalDataPath: opts.UpdateBundlePath,
+	}
+
+	go m.Start(ctx)
+
 	return l, nil
 }
 

libindex/options.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/updater/driver/v1"
 )
 
 const (
@@ -49,4 +50,9 @@ type Options struct {
 		Package, Dist, Repo, File map[string]func(interface{}) error
 	}
 	Resolvers []indexer.Resolver
+	// Updater specific configs
+	UpdaterConfigs   map[string]driver.ConfigUnmarshaler
+	UpdateInterval   time.Duration
+	UpdateBundleURL  string
+	UpdateBundlePath string
 }