Merge pull request #448 from gallettilance/force-bundle

Bug 1880501: Adding overwrite-latest flag to index add

cmd/opm/index/add.go

@@ -62,6 +62,10 @@ func addIndexAddCmd(parent *cobra.Command) {
 	indexCmd.Flags().Bool("permissive", false, "allow registry load errors")
 	indexCmd.Flags().StringP("mode", "", "replaces", "graph update mode that defines how channel graphs are updated. One of: [replaces, semver, semver-skippatch]")
 
+	indexCmd.Flags().Bool("overwrite-latest", false, "overwrite the latest bundles (channel heads) with those of the same csv name given by --bundles")
+	if err := indexCmd.Flags().MarkHidden("overwrite-latest"); err != nil {
+		logrus.Panic(err.Error())
+	}
 	if err := indexCmd.Flags().MarkHidden("debug"); err != nil {
 		logrus.Panic(err.Error())
 	}
@@ -118,6 +122,11 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	overwrite, err := cmd.Flags().GetBool("overwrite-latest")
+	if err != nil {
+		return err
+	}
+
 	modeEnum, err := registry.GetModeFromString(mode)
 	if err != nil {
 		return err
@@ -147,6 +156,7 @@ func runIndexAddCmdFunc(cmd *cobra.Command, args []string) error {
 		Permissive:        permissive,
 		Mode:              modeEnum,
 		SkipTLS:           skipTLS,
+		Overwrite:         overwrite,
 	}
 
 	err = indexAdder.AddToIndex(request)

cmd/opm/registry/add.go

@@ -73,6 +73,7 @@ func addFunc(cmd *cobra.Command, args []string) error {
 		Bundles:       bundleImages,
 		Mode:          modeEnum,
 		ContainerTool: containertools.NewContainerTool(containerTool, containertools.NoneTool),
+		Overwrite:     false,
 	}
 
 	logger := logrus.WithFields(logrus.Fields{"bundles": bundleImages})

pkg/lib/indexer/indexer.go

@@ -65,6 +65,7 @@ type AddToIndexRequest struct {
 	Mode              pregistry.Mode
 	CaFile            string
 	SkipTLS           bool
+	Overwrite         bool
 }
 
 // AddToIndex is an aggregate API used to generate a registry index image with additional bundles
@@ -88,6 +89,7 @@ func (i ImageIndexer) AddToIndex(request AddToIndexRequest) error {
 		Mode:          request.Mode,
 		SkipTLS:       request.SkipTLS,
 		ContainerTool: i.PullTool,
+		Overwrite:     request.Overwrite,
 	}
 
 	// Add the bundles to the registry

pkg/lib/registry/registry.go

@@ -31,10 +31,11 @@ type AddToRegistryRequest struct {
 	Bundles       []string
 	Mode          registry.Mode
 	ContainerTool containertools.ContainerTool
+	Overwrite     bool
 }
 
 func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
-	db, err := sql.Open("sqlite3", request.InputDatabase)
+	db, err := sql.Open("sqlite3", "file:"+request.InputDatabase+"?_foreign_keys=on")
 	if err != nil {
 		return err
 	}
@@ -84,51 +85,115 @@ func (r RegistryUpdater) AddToRegistry(request AddToRegistryRequest) error {
 		simpleRefs = append(simpleRefs, image.SimpleReference(ref))
 	}
 
-	if err := populate(context.TODO(), dbLoader, graphLoader, dbQuerier, reg, simpleRefs, request.Mode); err != nil {
+	if err := populate(context.TODO(), dbLoader, graphLoader, dbQuerier, reg, simpleRefs, request.Mode, request.Overwrite); err != nil {
 		r.Logger.Debugf("unable to populate database: %s", err)
 
 		if !request.Permissive {
 			r.Logger.WithError(err).Error("permissive mode disabled")
 			return err
-		} else {
-			r.Logger.WithError(err).Warn("permissive mode enabled")
 		}
+		r.Logger.WithError(err).Warn("permissive mode enabled")
 	}
 
 	return nil
 }
 
-func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, querier registry.Query, reg image.Registry, refs []image.Reference, mode registry.Mode) error {
+func unpackImage(ctx context.Context, reg image.Registry, ref image.Reference) (image.Reference, string, func(), error) {
 	var errs []error
+	workingDir, err := ioutil.TempDir("./", "bundle_tmp")
+	if err != nil {
+		errs = append(errs, err)
+	}
 
-	unpackedImageMap := make(map[image.Reference]string, 0)
-	for _, ref := range refs {
-		workingDir, err := ioutil.TempDir("./", "bundle_tmp")
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
-		defer os.RemoveAll(workingDir)
+	if err = reg.Pull(ctx, ref); err != nil {
+		errs = append(errs, err)
+	}
 
-		if err = reg.Pull(ctx, ref); err != nil {
-			errs = append(errs, err)
-			continue
-		}
+	if err = reg.Unpack(ctx, ref, workingDir); err != nil {
+		errs = append(errs, err)
+	}
 
-		if err = reg.Unpack(ctx, ref, workingDir); err != nil {
-			errs = append(errs, err)
-			continue
+	cleanup := func() {
+		if err := os.RemoveAll(workingDir); err != nil {
+			logrus.Error(err)
 		}
-
-		unpackedImageMap[ref] = workingDir
 	}
 
 	if len(errs) > 0 {
-		return utilerrors.NewAggregate(errs)
+		return nil, "", cleanup, utilerrors.NewAggregate(errs)
+	}
+	return ref, workingDir, cleanup, nil
+}
+
+func populate(ctx context.Context, loader registry.Load, graphLoader registry.GraphLoader, querier registry.Query, reg image.Registry, refs []image.Reference, mode registry.Mode, overwrite bool) error {
+	unpackedImageMap := make(map[image.Reference]string, 0)
+	for _, ref := range refs {
+		to, from, cleanup, err := unpackImage(ctx, reg, ref)
+		if err != nil {
+			return err
+		}
+		unpackedImageMap[to] = from
+		defer cleanup()
 	}
 
-	populator := registry.NewDirectoryPopulator(loader, graphLoader, querier, unpackedImageMap)
+	overwriteImageMap := make(map[string]map[image.Reference]string, 0)
+	if overwrite {
+		// find all bundles that are attempting to overwrite
+		for to, from := range unpackedImageMap {
+			img, err := registry.NewImageInput(to, from)
+			if err != nil {
+				return err
+			}
+			overwritten, err := querier.GetBundlePathIfExists(ctx, img.Bundle.Name)
+			if err != nil {
+				if err == registry.ErrBundleImageNotInDatabase {
+					continue
+				}
+				return err
+			}
+			if overwritten != "" {
+				// get all bundle paths for that package - we will re-add these to regenerate the graph
+				bundles, err := querier.GetBundlesForPackage(ctx, img.Bundle.Package)
+				if err != nil {
+					return err
+				}
+				cleanups := make(chan func(), 1)
+				errs := make(chan error, 1)
+				for bundle := range bundles {
+					if _, ok := overwriteImageMap[img.Bundle.Package]; !ok {
+						overwriteImageMap[img.Bundle.Package] = make(map[image.Reference]string, 0)
+					}
+					// parallelize image pulls
+					go func(bundle registry.BundleKey, img *registry.ImageInput) {
+						if bundle.CsvName != img.Bundle.Name {
+							to, from, cleanup, err := unpackImage(ctx, reg, image.SimpleReference(bundle.BundlePath))
+							if err != nil {
+								errs <- err
+							}
+							cleanups <- cleanup
+							overwriteImageMap[img.Bundle.Package][to] = from
+						} else {
+							overwriteImageMap[img.Bundle.Package][to] = from
+							delete(unpackedImageMap, to)
+						}
+					}(bundle, img)
+				}
+				for i := 0; i < len(bundles)-1; i++ {
+					select {
+					case err := <-errs:
+						return err
+					default:
+						cleanup := <-cleanups
+						defer cleanup()
+					}
+				}
+			} else {
+				return fmt.Errorf("index add --overwrite-latest is only supported when using bundle images")
+			}
+		}
+	}
 
+	populator := registry.NewDirectoryPopulator(loader, graphLoader, querier, unpackedImageMap, overwriteImageMap, overwrite)
 	return populator.Populate(mode)
 }
 

pkg/registry/bundle.go

@@ -73,7 +73,6 @@ func NewBundleFromStrings(name, pkgName string, channels []string, objs []string
 func (b *Bundle) Size() int {
 	return len(b.Objects)
 }
-
 func (b *Bundle) Add(obj *unstructured.Unstructured) {
 	b.Objects = append(b.Objects, obj)
 	b.cacheStale = true

pkg/registry/empty.go

@@ -104,6 +104,10 @@ func (EmptyQuery) GetDependenciesForBundle(ctx context.Context, name, version, p
 	return nil, errors.New("empty querier: cannot get dependencies for bundle")
 }
 
+func (EmptyQuery) GetBundlePathIfExists(ctx context.Context, csvName string) (bundlePath string, err error) {
+	return "", errors.New("empty querier: cannot get bundle path for bundle")
+}
+
 var _ Query = &EmptyQuery{}
 
 func NewEmptyQuerier() *EmptyQuery {

pkg/registry/imageinput.go

@@ -16,9 +16,9 @@ type ImageInput struct {
 	metadataDir      string
 	to               image.Reference
 	from             string
-	annotationsFile  *AnnotationsFile
+	AnnotationsFile  *AnnotationsFile
 	dependenciesFile *DependenciesFile
-	bundle           *Bundle
+	Bundle           *Bundle
 }
 
 func NewImageInput(to image.Reference, from string) (*ImageInput, error) {
@@ -72,7 +72,7 @@ func NewImageInput(to image.Reference, from string) (*ImageInput, error) {
 		metadataDir:      metadata,
 		to:               to,
 		from:             from,
-		annotationsFile:  annotationsFile,
+		AnnotationsFile:  annotationsFile,
 		dependenciesFile: dependenciesFile,
 	}
 
@@ -115,14 +115,14 @@ func (i *ImageInput) getBundleFromManifests() error {
 	bundle.Dependencies = i.dependenciesFile.GetDependencies()
 
 	bundle.Name = csvName
-	bundle.Package = i.annotationsFile.Annotations.PackageName
-	bundle.Channels = strings.Split(i.annotationsFile.Annotations.Channels, ",")
+	bundle.Package = i.AnnotationsFile.Annotations.PackageName
+	bundle.Channels = strings.Split(i.AnnotationsFile.Annotations.Channels, ",")
 
 	if err := bundle.AllProvidedAPIsInBundle(); err != nil {
 		return fmt.Errorf("error checking provided apis in bundle %s: %s", bundle.Name, err)
 	}
 
-	i.bundle = bundle
+	i.Bundle = bundle
 
 	return nil
 }

pkg/registry/interface.go

@@ -57,6 +57,8 @@ type Query interface {
 	ListBundles(ctx context.Context) (bundles []*api.Bundle, err error)
 	// Get the list of dependencies for a bundle
 	GetDependenciesForBundle(ctx context.Context, name, version, path string) (dependencies []*api.Dependency, err error)
+	// Get the bundle path if it exists
+	GetBundlePathIfExists(ctx context.Context, csvName string) (string, error)
 }
 
 // GraphLoader generates a graph