(fix) catalog deletion resilience support

Enables installed extensions to continue working when their source
catalog becomes unavailable or is deleted. When resolution fails due
to catalog unavailability, the operator now continues reconciling with
the currently installed bundle instead of failing.

Changes:
- Resolution falls back to installed bundle when catalog unavailable
- Unpacking skipped when maintaining current installed state
- Helm and Boxcutter appliers handle nil contentFS gracefully
- Version upgrades properly blocked without catalog access

This ensures workloads remain stable and operational even when the
catalog they were installed from is temporarily unavailable or deleted,
while appropriately preventing version changes that require catalog access.

Author: camilamacedo86

internal/operator-controller/applier/boxcutter.go

@@ -317,22 +317,34 @@ func (bc *Boxcutter) createOrUpdate(ctx context.Context, obj client.Object) erro
 }
 
 func (bc *Boxcutter) apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels, revisionAnnotations map[string]string) (bool, string, error) {
-	// Generate desired revision
-	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
+	// List all existing revisions
+	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
 	if err != nil {
 		return false, "", err
 	}
 
-	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
-		return false, "", fmt.Errorf("set ownerref: %w", err)
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, we should use the existing installed revision without generating a new one.
+	if contentFS == nil {
+		if len(existingRevisions) == 0 {
+			return false, "", fmt.Errorf("no bundle content available and no existing revisions found")
+		}
+		// Use the most recent revision and just reconcile it (don't create a new one)
+		// Boxcutter's ClusterExtensionRevision reconciliation will handle maintaining the resources
+		// Return success since we're maintaining the current state
+		return true, "", nil
 	}
 
-	// List all existing revisions
-	existingRevisions, err := bc.getExistingRevisions(ctx, ext.GetName())
+	// Generate desired revision
+	desiredRevision, err := bc.RevisionGenerator.GenerateRevision(ctx, contentFS, ext, objectLabels, revisionAnnotations)
 	if err != nil {
 		return false, "", err
 	}
 
+	if err := controllerutil.SetControllerReference(ext, desiredRevision, bc.Scheme); err != nil {
+		return false, "", fmt.Errorf("set ownerref: %w", err)
+	}
+
 	currentRevision := &ocv1.ClusterExtensionRevision{}
 	state := StateNeedsInstall
 	// check if we can update the current revision.

internal/operator-controller/applier/helm.go

@@ -103,6 +103,16 @@ func (h *Helm) runPreAuthorizationChecks(ctx context.Context, ext *ocv1.ClusterE
 }
 
 func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExtension, objectLabels map[string]string, storageLabels map[string]string) (bool, string, error) {
+	// If contentFS is nil, we're maintaining the current state without catalog access.
+	// In this case, reconcile the existing Helm release if it exists.
+	if contentFS == nil {
+		ac, err := h.ActionClientGetter.ActionClientFor(ctx, ext)
+		if err != nil {
+			return false, "", err
+		}
+		return h.reconcileExistingRelease(ctx, ac, ext)
+	}
+
 	chrt, err := h.buildHelmChart(contentFS, ext)
 	if err != nil {
 		return false, "", err
@@ -197,6 +207,40 @@ func (h *Helm) Apply(ctx context.Context, contentFS fs.FS, ext *ocv1.ClusterExte
 	return true, "", nil
 }
 
+// reconcileExistingRelease reconciles an existing Helm release without catalog access.
+// This is used when the catalog is unavailable but we need to maintain the current installation.
+// It reconciles the release and sets up watchers to ensure resources are maintained.
+func (h *Helm) reconcileExistingRelease(ctx context.Context, ac helmclient.ActionInterface, ext *ocv1.ClusterExtension) (bool, string, error) {
+	rel, err := ac.Get(ext.GetName())
+	if errors.Is(err, driver.ErrReleaseNotFound) {
+		return false, "", fmt.Errorf("no bundle content available and no existing release found")
+	}
+	if err != nil {
+		return false, "", fmt.Errorf("getting current release: %w", err)
+	}
+
+	// Reconcile the existing release to ensure resources are maintained
+	if err := ac.Reconcile(rel); err != nil {
+		return false, "", err
+	}
+
+	// Watch the release objects to ensure they're maintained
+	relObjects, err := util.ManifestObjects(strings.NewReader(rel.Manifest), fmt.Sprintf("%s-release-manifest", rel.Name))
+	if err != nil {
+		return true, "", err
+	}
+	klog.FromContext(ctx).Info("watching managed objects")
+	cache, err := h.Manager.Get(ctx, ext)
+	if err != nil {
+		return true, "", err
+	}
+	if err := cache.Watch(ctx, h.Watcher, relObjects...); err != nil {
+		return true, "", err
+	}
+
+	return true, "", nil
+}
+
 func (h *Helm) buildHelmChart(bundleFS fs.FS, ext *ocv1.ClusterExtension) (*chart.Chart, error) {
 	if h.HelmChartProvider == nil {
 		return nil, errors.New("HelmChartProvider is nil")

internal/operator-controller/controllers/clusterextension_reconcile_steps.go

@@ -95,11 +95,7 @@ func ResolveBundle(r resolve.Resolver) ReconcileStepFunc {
 			}
 			resolvedBundle, resolvedBundleVersion, resolvedDeprecation, err := r.Resolve(ctx, ext, bm)
 			if err != nil {
-				// Note: We don't distinguish between resolution-specific errors and generic errors
-				setStatusProgressing(ext, err)
-				setInstalledStatusFromRevisionStates(ext, state.revisionStates)
-				ensureAllConditionsWithReason(ext, ocv1.ReasonFailed, err.Error())
-				return nil, err
+				return handleResolutionError(ctx, state, ext, err)
 			}
 
 			// set deprecation status after _successful_ resolution
@@ -134,9 +130,41 @@ func ResolveBundle(r resolve.Resolver) ReconcileStepFunc {
 	}
 }
 
+// handleResolutionError handles the case when bundle resolution fails.
+// If a bundle is already installed, we fall back to using it to maintain the current state.
+// This enables workload resilience when the catalog becomes unavailable.
+func handleResolutionError(ctx context.Context, state *reconcileState, ext *ocv1.ClusterExtension, err error) (*ctrl.Result, error) {
+	l := log.FromContext(ctx)
+
+	// If we have an installed bundle, fall back to it to maintain current state
+	if state.revisionStates.Installed != nil {
+		l.Info("resolution failed but continuing with installed bundle", "error", err, "installedBundle", state.revisionStates.Installed.BundleMetadata)
+		setStatusProgressing(ext, err)
+		setInstalledStatusFromRevisionStates(ext, state.revisionStates)
+		state.resolvedRevisionMetadata = state.revisionStates.Installed
+		return nil, nil
+	}
+
+	// No installed bundle and resolution failed - cannot proceed
+	setStatusProgressing(ext, err)
+	setInstalledStatusFromRevisionStates(ext, state.revisionStates)
+	ensureAllConditionsWithReason(ext, ocv1.ReasonFailed, err.Error())
+	return nil, err
+}
+
 func UnpackBundle(i imageutil.Puller, cache imageutil.Cache) ReconcileStepFunc {
 	return func(ctx context.Context, state *reconcileState, ext *ocv1.ClusterExtension) (*ctrl.Result, error) {
 		l := log.FromContext(ctx)
+
+		// Skip unpacking if we're using an already-installed bundle
+		// (e.g., when catalog is unavailable but we're maintaining current state)
+		if state.revisionStates.Installed != nil &&
+			state.resolvedRevisionMetadata.BundleMetadata == state.revisionStates.Installed.BundleMetadata {
+			l.Info("skipping unpack - using installed bundle content")
+			// imageFS will remain nil - the applier will use the existing installed content
+			return nil, nil
+		}
+
 		l.Info("unpacking resolved bundle")
 		imageFS, _, _, err := i.Pull(ctx, ext.GetName(), state.resolvedRevisionMetadata.Image, cache)
 		if err != nil {